投屏Sink端音频底层解码并用OpenSLES进行播放
一、代码分析
在公司项目中，音频解码及播放是把数据传到Java层进行解码播放的，其实这个步骤没有必要，完全可以在底层进行处理。
通过代码发现其实也做了在底层进行解码，那么为啥不直接使用底层解码播放呢，我们可以看看原先代码怎么做的：
代码中通过定义的宏DECODE_AUDIO_IN_JAVA来控制mAudioCodec对象是否创建，然后在通过mAudioCodec对象是否为null来控制音频数据是否传给Java层处理，代码中原来已经支持了在底层解码然后在传回上传使用AudioTrack进行播放，那我求改宏DECODE_AUDIO_IN_JAVA来让其在底层进行解码，运行后会发现播放的声音非常的卡顿。
二、解决办法
最终发现原来是在使用底层处理时播放的音频是数据大小传的不对，会导致播放的声音非常的卡顿。
解决办法就是将下面红框的修改成info.size就可以了。
三、底层播放音频
但是这样还是将音频的播放传给Java层进行播放。
我们可以通过使用OpenSLES来处理底层音频的播放
3.1 OpenSLRender类的实现
#ifndef _OPENSLRENDER_HEAD_
#define _OPENSLRENDER_HEAD_
#include <media/stagefright/foundation/ABuffer.h>
#include <media/stagefright/foundation/ADebug.h>
#include <media/stagefright/foundation/AMessage.h>
#include <SLES/OpenSLES.h>
#include <SLES/OpenSLES_Android.h>
namespace android{
class OpenSLRender : public Thread {
public:
OpenSLRender(int64_t buffertime,int32_t bufferframes=5);
~OpenSLRender();
bool init(int32_t chanNum,int rate);
void stop();
void setBufferTimes(int64_t buffertime);
void queueInputBuffer(sp<ABuffer> data);
void playerCallback();
private:
SLAndroidSimpleBufferQueueItf bqPlayerBufferQueue;
SLObjectItf bqPlayerObject;
SLPlayItf bqPlayerPlay;
SLObjectItf outputMixObject;
SLObjectItf engineObject;
List<sp<ABuffer>> mList;
int64_t mBufferTimeUs;
int32_t mBufferFrames;
int64_t mLasPts;
bool bFist;
pthread_mutex_t startMutex;
pthread_cond_t startCond;
Mutex dataMutex; // for data in/out on diffrent thread
bool bRun;
sp<ABuffer> mMuteData;
int64_t mlastAudtime;
int mPlayAudFrames;
int mDropFrames;
int32_t muteCounts;
sp<ABuffer> mRenderData;
int32_t mOverFlowContinuetimes;
private:
virtual bool threadLoop();
sp<ABuffer> intervalOut(int gap);
sp<ABuffer> dropToMaxBuffer(int gap);
sp<ABuffer> dropAutoNums();
sp<ABuffer> getNextBuffer();
void destroy();
static void bqPlayerCallback(SLAndroidSimpleBufferQueueItf bq, void *context);
DISALLOW_EVIL_CONSTRUCTORS(OpenSLRender);
};
}
#endif
//#define LOG_NDEBUG 0
#define LOG_TAG "OpenSLRender"
#include"OpenSLRender.h"
#define UNUSED(x) ((void)x)
#define AUD_DROP_THRESHOLD 5
namespace android{
OpenSLRender::OpenSLRender(int64_t bufferTime,int32_t bufferFrames):
mBufferTimeUs(bufferTime),
mBufferFrames(bufferFrames),
bFist(true),
startMutex(PTHREAD_MUTEX_INITIALIZER),
startCond(PTHREAD_COND_INITIALIZER),
bRun(true),
mRenderData(NULL),
mMuteData(new ABuffer(2048)),
muteCounts(0),
mlastAudtime(0),
mPlayAudFrames(0),
mDropFrames(0),
mOverFlowContinuetimes(0){
memset(mMuteData->data(),0,mMuteData->size());
}
OpenSLRender::~OpenSLRender(){
stop();
requestExit();
requestExitAndWait();
//this.clear(); //sp<>.clear, this is not sp
}
bool OpenSLRender::init(int32_t chanNum,int rate){
// engine interfaces
SLEngineItf engineEngine;
// output mix interfaces
SLEnvironmentalReverbItf outputMixEnvironmentalReverb = NULL;
// aux effect on the output mix, used by the buffer queue player
const SLEnvironmentalReverbSettings reverbSettings =
SL_I3DL2_ENVIRONMENT_PRESET_DEFAULT;
// buffer queue player interfaces
SLresult result;
// create engine
result = slCreateEngine(&engineObject, 0, NULL, 0, NULL, NULL);
CHECK(SL_RESULT_SUCCESS == result);
(void)result;
// realize the engine
result = (*engineObject)->Realize(engineObject, SL_BOOLEAN_FALSE);
CHECK(SL_RESULT_SUCCESS == result);
(void)result;
// get the engine interface, which is needed in order to create other objects
result = (*engineObject)->GetInterface(engineObject, SL_IID_ENGINE, &engineEngine);
CHECK(SL_RESULT_SUCCESS == result);
(void)result;
// create output mix, with environmental reverb specified as a non-required interface
const SLInterfaceID ids[1] = {SL_IID_ENVIRONMENTALREVERB};
const SLboolean req[1] = {SL_BOOLEAN_FALSE};
result = (*engineEngine)->CreateOutputMix(engineEngine, &outputMixObject, 1, ids, req);
CHECK(SL_RESULT_SUCCESS == result);
(void)result;
// realize the output mix
result = (*outputMixObject)->Realize(outputMixObject, SL_BOOLEAN_FALSE);
CHECK(SL_RESULT_SUCCESS == result);
(void)result;
// get the environmental reverb interface
// this could fail if the environmental reverb effect is not available,
// either because the feature is not present, excessive CPU load, or
// the required MODIFY_AUDIO_SETTINGS permission was not requested and granted
result = (*outputMixObject)->GetInterface(outputMixObject, SL_IID_ENVIRONMENTALREVERB,
&outputMixEnvironmentalReverb);
if (SL_RESULT_SUCCESS == result) {
result = (*outputMixEnvironmentalReverb)->SetEnvironmentalReverbProperties(
outputMixEnvironmentalReverb, &reverbSettings);
(void)result;
}
// ignore unsuccessful result codes for environmental reverb, as it is optional for this example
{
// configure audio source
SLuint32 samplesPerSec = SL_SAMPLINGRATE_48;
if(48000 == rate){
samplesPerSec = SL_SAMPLINGRATE_48;
}else if(44100 == rate){
samplesPerSec = SL_SAMPLINGRATE_44_1;
}
SLuint32 audChan = chanNum;
SLDataLocator_AndroidSimpleBufferQueue loc_bufq = {SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE, 2};
SLDataFormat_PCM format_pcm = {SL_DATAFORMAT_PCM,
(audChan == 0) ? 2 : audChan,
samplesPerSec,
SL_PCMSAMPLEFORMAT_FIXED_16,
SL_PCMSAMPLEFORMAT_FIXED_16,
SL_SPEAKER_FRONT_LEFT | SL_SPEAKER_FRONT_RIGHT,//
SL_BYTEORDER_LITTLEENDIAN};
/*
* Enable Fast Audio when possible: once we set the same rate to be the native, fast audio path
* will be triggered
*/
SLDataSource audioSrc = {&loc_bufq, &format_pcm};
// configure audio sink
SLDataLocator_OutputMix loc_outmix = {SL_DATALOCATOR_OUTPUTMIX, outputMixObject};
SLDataSink audioSnk = {&loc_outmix, NULL};
/*
* create audio player:
* fast audio does not support when SL_IID_EFFECTSEND is required, skip it
* for fast audio case
*/
const SLInterfaceID ids[3] = {SL_IID_BUFFERQUEUE, SL_IID_VOLUME,/* SL_IID_EFFECTSEND,
SL_IID_MUTESOLO,*/};
const SLboolean req[3] = {SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE,
/*SL_BOOLEAN_TRUE,*/ };
result = (*engineEngine)->CreateAudioPlayer(engineEngine, &bqPlayerObject, &audioSrc, &audioSnk,
2, ids, req);
CHECK(SL_RESULT_SUCCESS == result);
(void)result;
// realize the player
result = (*bqPlayerObject)->Realize(bqPlayerObject, SL_BOOLEAN_FALSE);
CHECK(SL_RESULT_SUCCESS == result);
(void)result;
// get the play interface
result = (*bqPlayerObject)->GetInterface(bqPlayerObject, SL_IID_PLAY, &bqPlayerPlay);
CHECK(SL_RESULT_SUCCESS == result);
(void)result;
// get the buffer queue interface
result = (*bqPlayerObject)->GetInterface(bqPlayerObject, SL_IID_BUFFERQUEUE,
&bqPlayerBufferQueue);
CHECK(SL_RESULT_SUCCESS == result);
(void)result;
// register callback on the buffer queue
result = (*bqPlayerBufferQueue)->RegisterCallback(bqPlayerBufferQueue, bqPlayerCallback, this);
CHECK(SL_RESULT_SUCCESS == result);
(void)result;
// set the player's state to playing
result = (*bqPlayerPlay)->SetPlayState(bqPlayerPlay, SL_PLAYSTATE_PLAYING);
CHECK(SL_RESULT_SUCCESS == result);
(void)result;
status_t err = run("opensl buffering", ANDROID_PRIORITY_AUDIO);
CHECK(err==OK);
return true;
}
}
void OpenSLRender:: destroy(){
ALOGE("opeslRender destroy ![%s][%d]",__FUNCTION__,__LINE__);
(*bqPlayerPlay)->SetPlayState(bqPlayerPlay, SL_PLAYSTATE_STOPPED);
// destroy buffer queue audio player object, and invalidate all associated interfaces
if (bqPlayerObject != NULL) {
(*bqPlayerObject)->Destroy(bqPlayerObject);
bqPlayerObject = NULL;
bqPlayerPlay = NULL;
bqPlayerBufferQueue = NULL;
}
// destroy output mix object, and invalidate all associated interfaces
if (outputMixObject != NULL) {
(*outputMixObject)->Destroy(outputMixObject);
outputMixObject = NULL;
}
// destroy engine object, and invalidate all associated interfaces
if (engineObject != NULL) {
(*engineObject)->Destroy(engineObject);
engineObject = NULL;
}
}
void OpenSLRender::stop(){
// AutoMutex _l(dataMutex);
ALOGE("OpenSLRender_stop:[%s%d]",__FUNCTION__,__LINE__);
if(bRun==true){
bRun=false;
destroy();
}
}
//to support Adjustment
void OpenSLRender::setBufferTimes(int64_t buffertime){
AutoMutex _l(dataMutex);
mBufferTimeUs = buffertime;
}
void OpenSLRender::queueInputBuffer(sp<ABuffer> data){
//input buffer, becareful!!!!!!!
AutoMutex _l(dataMutex);
//to chek pts
if(!mList.empty()){
sp<ABuffer> max = *(--mList.end());
int64_t dataPts=0,maxPts=0;
CHECK(data->meta()->findInt64("timePts", &dataPts));
CHECK(max->meta()->findInt64("timePts", &maxPts));
if(dataPts < maxPts){
ALOGD("[%s%d] pts erro data:%ld list:%ld\n",__FUNCTION__,__LINE__,maxPts,dataPts);
return;
}
}
#if ENABLE_STATISTICS
ALOGD(COMMON_DEBUG," Audio in, %lld remain __%ld__ [%s%d]\n",ALooper::GetNowUs(), mList.size(),__FUNCTION__,__LINE__);
#endif
mList.push_back(data);
if(bFist){
sp<ABuffer> min = *mList.begin();
sp<ABuffer> max = *(--mList.end());
int64_t minPts=0,maxPts=0;
CHECK(min->meta()->findInt64("timePts", &minPts));
CHECK(max->meta()->findInt64("timePts", &maxPts));
// ALOGE("==minPts=%lld,maxPts:%lld,mBufferTimeUs:%lld,(maxPts - minPts)=%lld",minPts,maxPts,mBufferTimeUs,(maxPts - minPts));
//if((maxPts - minPts > mBufferTimeUs) || mList.size()>=mBufferFrames){
if((maxPts - minPts > mBufferTimeUs/2) || mList.size()>=mBufferFrames/2){
//buffer over! go---------
pthread_mutex_lock(&startMutex);
pthread_cond_signal(&startCond);
pthread_mutex_unlock(&startMutex);
}
}
}
void OpenSLRender::playerCallback(){
AutoMutex _l(dataMutex);
if(!bRun){
return;
}
int64_t nowUs = ALooper::GetNowUs();
if(!mList.empty()){
sp<ABuffer> min = *mList.begin();
sp<ABuffer> max = *(--mList.end());
int64_t minPts=0,maxPts=0;
CHECK(min->meta()->findInt64("timePts", &minPts));
CHECK(max->meta()->findInt64("timePts", &maxPts));
//if(maxPts - minPts > mBufferTimeUs -timeDuration){
if(mList.size()>=mBufferFrames) {
mOverFlowContinuetimes++;
}else{
mOverFlowContinuetimes = 0;
}
if(mOverFlowContinuetimes > AUD_DROP_THRESHOLD)
{
//"Break out"
//Take one output to render for every two buffers
//data = intervalOut(2);
//data = dropAutoNums();
int flowFrames = mList.size() - mBufferFrames;
if( flowFrames >= mBufferFrames){
//ALOGD(COMMON_DEBUG,"video jetterbuff dopallflows %d [%s%d] mList.size():%ld \n",flowFrames,__FUNCTION__,__LINE__,mList.size());
sp<ABuffer> data = dropToMaxBuffer(flowFrames);
mRenderData = getNextBuffer();
}else{
//"Break out"
//Take one output to render for every two buffers
sp<ABuffer> data = dropToMaxBuffer(2);
mRenderData = getNextBuffer();
}
mOverFlowContinuetimes = 0;
}else{
//one by one
mRenderData = getNextBuffer();
mPlayAudFrames++;
}
}else{
mRenderData = mMuteData;
muteCounts++;
mOverFlowContinuetimes = 0;
}
if(mRenderData ==NULL){
//just give the mutex data
mRenderData = mMuteData;
muteCounts++;
}
SLresult result;
//enqueue another buffer
result = (*bqPlayerBufferQueue)->Enqueue(bqPlayerBufferQueue, mRenderData->data(), mRenderData->size());
// the most likely other result is SL_RESULT_BUFFER_INSUFFICIENT,
// which for this code example would indicate a programming error
if (SL_RESULT_SUCCESS != result) {
}
if(!mlastAudtime)
{
mlastAudtime = nowUs;
}
if(nowUs - mlastAudtime >= 1000*1000)
{
ALOGE("playback(%d) droped(%d) muteCounts(%d) frames in one second,QSize:%d",mPlayAudFrames,mDropFrames,muteCounts,(int32_t)mList.size());
mDropFrames = 0;
mPlayAudFrames = 0;
mlastAudtime = nowUs;
muteCounts = 0;
}
}
void OpenSLRender::bqPlayerCallback(SLAndroidSimpleBufferQueueItf bq, void *context){
UNUSED(bq);
OpenSLRender * pRender =static_cast<OpenSLRender*>(context);
if(pRender){
pRender->playerCallback();
}
}
sp<ABuffer> OpenSLRender::intervalOut(int gap){
int count =0;
sp<ABuffer> data = NULL;
while( (data = getNextBuffer())!=NULL && ++count < gap){
//ALOGD(COMMON_DEBUG," audio drop one [%s%d] remain mList.size():%ld \n",__FUNCTION__,__LINE__,mList.size());
}
return data;
}
sp<ABuffer> OpenSLRender::dropToMaxBuffer(int gap){
sp<ABuffer> data = NULL;
int count = 0;
while( (data = getNextBuffer())!=NULL && count++ < gap){
mDropFrames++;
//ALOGD(COMMON_DEBUG," audio drop one [%s%d] remain mList.size():%ld \n",__FUNCTION__,__LINE__,mList.size());
}
return data;
}
sp<ABuffer> OpenSLRender::dropAutoNums(){
sp<ABuffer> data = NULL;
while( (data = getNextBuffer())!=NULL && muteCounts>0){
muteCounts--;
//ALOGD(COMMON_DEBUG," audio drop one [%s%d] remain mList.size():%ld \n",__FUNCTION__,__LINE__,mList.size());
}
return data;
}
sp<ABuffer> OpenSLRender::getNextBuffer(){
if(!mList.empty()){
sp<ABuffer> data = *mList.begin();
mList.erase(mList.begin());
return data;
}
return NULL;
}
bool OpenSLRender::threadLoop(){
if(bFist){
pthread_mutex_lock(&startMutex);
pthread_cond_wait(&startCond,&startMutex);
pthread_mutex_unlock(&startMutex);
ALOGE("[%s%d]start out\n",__FUNCTION__,__LINE__);
bFist = false;
}
//to start play
playerCallback();
return false;
}
}
3.2 OpenSLRender类的使用
3.2.1 创建OpenSLRender对象并初始化
3.2.2 OpenSLRender的数据处理播放
3.2.3 OpenSLRender的停止
