java — отправляет аудиоданные в распознаватель речи ibm watson с помощью OpenSL ES

В настоящее время я использую OpenSL ES для получения аудиоданных с микрофона локального устройства. Я использую PipedStream для отправки аудиоданных в распознаватель речи ibm watson с помощью websocket, но я не получаю никаких результатов, и я не совсем уверен, почему. Мне нужна помощь, чтобы получить результаты с помощью микрофона.

Это мой MainActivity.java

PipedInputStream pipedInputStream;
PipedOutputStream pipedOutputStream;

MyDataHandler handler;

SpeechToText speechToTextService;
String result;
Boolean isFinal;

AudioFormat audioFormat;

@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
try {
pipedOutputStream = new PipedOutputStream();
pipedInputStream = new PipedInputStream(pipedOutputStream);
handler = new MyDataHandler(pipedOutputStream);

new Thread(new Runnable() {
@Override
public void run() {
start_process();
}
}).start();

new Thread(new Runnable() {
@Override
public void run() {
while (true) {
doCallToNativeFunction(pipedOutputStream);
try {
Thread.sleep(100);
} catch (InterruptedException e) {
Log.e("MainActivity", e.getMessage(), e);
}
}
}
}).start();

speechToTextService = new SpeechToText();
speechToTextService.setUsernameAndPassword("f1361112-2e3e-4787-b180-4df70aee83a1",
"BCouSNYeaqtc");

new Thread(new Runnable() {
@Override
public void run() {
RecognizeOptions options = new RecognizeOptions.Builder()
.continuous(true)
.interimResults(true)
.contentType(HttpMediaType.AUDIO_PCM + "; rate=" + 44100)
.inactivityTimeout(-1)
.wordConfidence(true)
.build();
speechToTextService.recognizeUsingWebSocket(pipedInputStream, options, new BaseRecognizeCallback() {
@Override
public void onTranscription(SpeechResults speechResults) {
try {
result = speechResults.getResults().get(0).getAlternatives()
.get(0).getTranscript();
isFinal = speechResults.getResults().get(0).isFinal();

System.out.println(speechResults);
} catch (Exception e) {
Log.e("MainActivity", e.getMessage(), e);
}
}
});
}
}).start();
} catch (IOException e) {
Log.e("MainActivity", e.getMessage(), e);
}
}

public void doCallToNativeFunction(OutputStream outputStream) {
getAudioDataWithHandler(handler);
}

DataHandler.java

OutputStream dataStream;

public MyDataHandler(OutputStream writeTo) {
dataStream = writeTo;
}
public void handleData(byte[] incomingData) {
try {
dataStream.write(incomingData, 0, incomingData.length);
dataStream.flush();
} catch (Exception e) {
Log.e("MyDataHandler", e.getMessage(), e);
}
}

MainSystem.cpp

OPENSL_STREAM *p;
int samps, i, j;
float inbuffer[VECSAMPS_MONO], outbuffer[VECSAMPS_STEREO];

static int on;
JNIEXPORT void JNICALL
start_process() {
p = android_OpenAudioDevice(SR, 1, 1, BUFFERFRAMES);
if (p == NULL) return;
}

JNIEXPORT void JNICALL
stop_process() {
on = 0;
}

JNIEXPORT void JNICALL
getAudioDataWithHandler(JNIEnv *env,
jclass
clazz,
jobject
obj) {

if (p != NULL) {
samps = android_AudioIn(p, inbuffer, VECSAMPS_MONO);

jbyteArray audioDataCopy = env->NewByteArray(samps);
env->SetByteArrayRegion(audioDataCopy, 0, samps, (jbyte *) inbuffer);

jmethodID aMethodId = env->GetMethodID(env->GetObjectClass(obj), "handleData", "([B)V");
assert(0 != aMethodId);
env->CallVoidMethod(obj, aMethodId, audioDataCopy);
} else {
__android_log_print(ANDROID_LOG_ERROR, "s", "NULL");
}
}

OpenSLSystem.cpp

static void *createThreadLock(void);
static void   waitThreadLock(void *lock);
static void  notifyThreadLock(void *lock);
static void  destroyThreadLock(void *lock);
static void  bqRecorderCallback(SLAndroidSimpleBufferQueueItf bq, void
*context);

static void openSLCreateEngine(OPENSL_STREAM *p) {
SLresult result;

result = slCreateEngine(&(p->engineObject), 0 , NULL, 0, NULL, NULL);
assert(result == SL_RESULT_SUCCESS);

result = (*p->engineObject)->Realize(p->engineObject, SL_BOOLEAN_FALSE);
assert(result == SL_RESULT_SUCCESS);

result = (*p->engineObject)->GetInterface(p->engineObject, SL_IID_ENGINE, &(p->engineEngine));
assert(result == SL_RESULT_SUCCESS);
}

static void openSLRecOpen(OPENSL_STREAM *p) {

SLresult result;
SLuint32 sr = p->sr;
SLuint32 channels = p->inchannels;

if (channels) {
switch(sr) {
case 8000:
sr = SL_SAMPLINGRATE_8;
break;
case 11025:
sr = SL_SAMPLINGRATE_11_025;
break;
case 16000:
sr = SL_SAMPLINGRATE_16;
break;
case 22050:
sr = SL_SAMPLINGRATE_22_05;
break;
case 24000:
sr = SL_SAMPLINGRATE_24;
break;
case 32000:
sr = SL_SAMPLINGRATE_32;
break;
case 44100:
sr = SL_SAMPLINGRATE_44_1;
break;
case 64000:
sr = SL_SAMPLINGRATE_64;
break;
case 88200:
sr = SL_SAMPLINGRATE_88_2;
break;
case 96000:
sr = SL_SAMPLINGRATE_96;
break;
case 192000:
sr = SL_SAMPLINGRATE_192;
break;
default:
return;
}

SLDataLocator_IODevice loc_dev = {SL_DATALOCATOR_IODEVICE, SL_IODEVICE_AUDIOINPUT,
SL_DEFAULTDEVICEID_AUDIOINPUT, NULL};
SLDataSource audioSrc = {&loc_dev, NULL};

SLuint32 speakers;
if (channels > 1)
speakers = SL_SPEAKER_FRONT_LEFT | SL_SPEAKER_FRONT_RIGHT;
else speakers = SL_SPEAKER_FRONT_CENTER;
SLDataLocator_AndroidSimpleBufferQueue loc_bq =
{SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE, 2};
SLDataFormat_PCM format_pcm = {SL_DATAFORMAT_PCM, channels, sr,
SL_PCMSAMPLEFORMAT_FIXED_8,
SL_PCMSAMPLEFORMAT_FIXED_8,
speakers, SL_BYTEORDER_LITTLEENDIAN};
SLDataSink audioSnk = {&loc_bq, &format_pcm};

const SLInterfaceID id[1] = {SL_IID_ANDROIDSIMPLEBUFFERQUEUE};
const SLboolean req[1] = {SL_BOOLEAN_FALSE};
result = (*p->engineEngine)->CreateAudioRecorder(p->engineEngine, &(p->recorderObject), &audioSrc,
&audioSnk, 1, id, req);
assert(result == SL_RESULT_SUCCESS);

result = (*p->recorderObject)->Realize(p->recorderObject, SL_BOOLEAN_FALSE);
assert(result == SL_RESULT_SUCCESS);

result = (*p->recorderObject)->GetInterface(p->recorderObject, SL_IID_RECORD, &(p->recorderRecord));
assert(result == SL_RESULT_SUCCESS);

result = (*p->recorderObject)->GetInterface(p->recorderObject, SL_IID_ANDROIDSIMPLEBUFFERQUEUE,
&(p->recorderBufferQueue));
assert(result == SL_RESULT_SUCCESS);

result = (*p->recorderBufferQueue)->RegisterCallback(p->recorderBufferQueue, bqRecorderCallback,
p);
assert(result == SL_RESULT_SUCCESS);

result = (*p->recorderRecord)->SetRecordState(p->recorderRecord, SL_RECORDSTATE_RECORDING);
assert(result == SL_RESULT_SUCCESS);
}
}

static void openSLDestroyEngine(OPENSL_STREAM *p) {
if (p->recorderObject != NULL) {
(*p->recorderObject)->Destroy(p->recorderObject);
p->recorderObject = NULL;
p->recorderRecord = NULL;
p->recorderBufferQueue = NULL;
}

if (p->engineObject != NULL) {
(*p->engineObject)->Destroy(p->engineObject);
p->engineObject = NULL;
p->engineEngine = NULL;
}
}
OPENSL_STREAM *android_OpenAudioDevice(int sr, int inchannels, int outchannels, int bufferframes) {
OPENSL_STREAM *p;
p = (OPENSL_STREAM *) calloc(sizeof(OPENSL_STREAM), 1);

p->inchannels = inchannels;
p->outchannels = outchannels;
p->sr = sr;
p->inlock = createThreadLock();
p->outlock = createThreadLock();

if ((p->outBufSamples = bufferframes * outchannels) != 0) {
if ((p->outputBuffer[0] = (short *) calloc(p->outBufSamples, sizeof(short))) == NULL ||
(p->outputBuffer[1] = (short *) calloc(p->outBufSamples, sizeof(short))) == NULL) {
android_CloseAudioDevice(p);
return NULL;
}
}

if ((p->inBufSamples = bufferframes * inchannels) != 0) {
if ((p->inputBuffer[0] = (short *) calloc(p->inBufSamples, sizeof(short))) == NULL ||
(p->inputBuffer[1] = (short *) calloc(p->inBufSamples, sizeof(short))) == NULL) {
android_CloseAudioDevice(p);
return NULL;
}
}

p->currentInputIndex = 0;
p->currentOutputBuffer = 0;
p->currentInputIndex = p->inBufSamples;
p->currentInputBuffer = 0;

openSLCreateEngine(p);
openSLRecOpen(p);

notifyThreadLock(p->outlock);
notifyThreadLock(p->inlock);

p->time = 0.;
return p;
}

void android_CloseAudioDevice(OPENSL_STREAM *p) {
if (p == NULL)
return;

openSLDestroyEngine(p);

if (p->inlock != NULL) {
notifyThreadLock(p->inlock);
destroyThreadLock(p->inlock);
p->inlock = NULL;
}

if (p->outlock != NULL) {
notifyThreadLock(p->outlock);
destroyThreadLock(p->outlock);
p->outlock = NULL;
}

if (p->outputBuffer[0] != NULL) {
free(p->outputBuffer[0]);
p->outputBuffer[0] = NULL;
}

if (p->outputBuffer[1] != NULL) {
free(p->outputBuffer[1]);
p->outputBuffer[1] = NULL;
}

if (p->inputBuffer[0] != NULL) {
free(p->inputBuffer[0]);
p->inputBuffer[0] = NULL;
}

if (p->inputBuffer[1] != NULL) {
free(p->inputBuffer[1]);
p->inputBuffer[1] = NULL;
}

free(p);
}

void bqRecorderCallback(SLAndroidSimpleBufferQueueItf bq, void *context) {
OPENSL_STREAM *p = (OPENSL_STREAM *) context;
notifyThreadLock(p->inlock);
}

int android_AudioIn(OPENSL_STREAM *p, float *buffer, int size) {
short *inBuffer;
int i, bufsamps = p->inBufSamples, index = p->currentInputIndex;
if (p == NULL || bufsamps == 0) return 0;

inBuffer = p->inputBuffer[p->currentInputBuffer];
for (i = 0; i < size; i++) {
if (index >= bufsamps) {
waitThreadLock(p->inlock);
(*p->recorderBufferQueue)->Enqueue(p->recorderBufferQueue,
inBuffer, bufsamps * sizeof(short));
p->currentInputBuffer = (p->currentInputBuffer ? 0 : 1);
index = 0;
inBuffer = p->inputBuffer[p->currentInputBuffer];
}
buffer[i] = (float) inBuffer[index++] *CONVMYFLT;
}
p->currentInputIndex = index;
if (p->outchannels == 0) p->time += (double) size / (p->sr * p->inchannels);
return i;
}
void *createThreadLock(void) {
threadLock *p;
p = (threadLock *) malloc(sizeof(threadLock));
if (p == NULL)
return NULL;
memset(p, 0, sizeof(threadLock));
if (pthread_mutex_init(&(p->m), (pthread_mutexattr_t *) NULL) != 0) {
free((void *) p);
return NULL;
}
if (pthread_cond_init(&(p->c), (pthread_condattr_t *) NULL) != 0) {
pthread_mutex_destroy(&(p->m));
free((void *) p);
return NULL;
}
p->s = (unsigned char) 1;

return p;
}void waitThreadLock(void *lock) {
threadLock *p;
int __unused retval = 0;
p = (threadLock *) lock;
pthread_mutex_lock(&(p->m));
while (!p->s) {
pthread_cond_wait(&(p->c), &(p->m));
}
p->s = (unsigned char) 0;
pthread_mutex_unlock(&(p->m));
}
void notifyThreadLock(void *lock) {
threadLock *p;
p = (threadLock *) lock;
pthread_mutex_lock(&(p->m));
p->s = (unsigned char) 1;
pthread_cond_signal(&(p->c));
pthread_mutex_unlock(&(p->m));
}

void destroyThreadLock(void *lock) {
threadLock *p;
p = (threadLock *) lock;
if (p == NULL)
return;
notifyThreadLock(p);
pthread_cond_destroy(&(p->c));
pthread_mutex_destroy(&(p->m));
free(p);
}

Update01

Таким образом, я воспроизводил данные на динамики моего устройства, и он просто воспроизводит кучу щелкающих звуков, так что это определенно сводится к аудиоданным.

Update02

Я распечатал массив, пока он считывал аудиоданные, и он печатает тот же буфер. Так что я предполагаю, что структура моей системы по какой-то причине не получает новый буфер от микрофона.