Source code
Revision control
Copy as Markdown
Other Tools
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set ts=2 sw=2 sts=2 et cindent: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
#include "nsThreadUtils.h"
#include "nsXPCOMCIDInternal.h"
#include "OnlineSpeechRecognitionService.h"
#include "nsIFile.h"
#include "SpeechGrammar.h"
#include "SpeechRecognition.h"
#include "SpeechRecognitionAlternative.h"
#include "SpeechRecognitionResult.h"
#include "SpeechRecognitionResultList.h"
#include "nsIObserverService.h"
#include "mozilla/dom/Document.h"
#include "mozilla/Preferences.h"
#include "mozilla/ScopeExit.h"
#include "mozilla/StaticPrefs_media.h"
#include "mozilla/Services.h"
#include "nsDirectoryServiceDefs.h"
#include "nsDirectoryServiceUtils.h"
#include "nsNetUtil.h"
#include "nsContentUtils.h"
#include "nsIChannel.h"
#include "nsIHttpChannel.h"
#include "nsIPrincipal.h"
#include "nsIStreamListener.h"
#include "nsIUploadChannel2.h"
#include "mozilla/dom/ClientIPCTypes.h"
#include "nsStringStream.h"
#include "nsIOutputStream.h"
#include "nsStreamUtils.h"
#include "OpusTrackEncoder.h"
#include "OggWriter.h"
#include "nsIClassOfService.h"
#include <json/json.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
namespace mozilla {
using namespace dom;
#define PREFERENCE_DEFAULT_RECOGNITION_ENDPOINT \
"media.webspeech.service.endpoint"
#define MAX_LISTENING_TIME_MS 10000
NS_IMPL_ISUPPORTS(OnlineSpeechRecognitionService, nsISpeechRecognitionService,
nsIStreamListener)
NS_IMETHODIMP
OnlineSpeechRecognitionService::OnStartRequest(nsIRequest* aRequest) {
MOZ_ASSERT(NS_IsMainThread());
return NS_OK;
}
static nsresult AssignResponseToBuffer(nsIInputStream* aIn, void* aClosure,
const char* aFromRawSegment,
uint32_t aToOffset, uint32_t aCount,
uint32_t* aWriteCount) {
nsCString* buf = static_cast<nsCString*>(aClosure);
buf->Append(aFromRawSegment, aCount);
*aWriteCount = aCount;
return NS_OK;
}
NS_IMETHODIMP
OnlineSpeechRecognitionService::OnDataAvailable(nsIRequest* aRequest,
nsIInputStream* aInputStream,
uint64_t aOffset,
uint32_t aCount) {
MOZ_ASSERT(NS_IsMainThread());
nsresult rv;
uint32_t readCount;
rv = aInputStream->ReadSegments(AssignResponseToBuffer, &mBuf, aCount,
&readCount);
NS_ENSURE_SUCCESS(rv, rv);
return NS_OK;
}
NS_IMETHODIMP
OnlineSpeechRecognitionService::OnStopRequest(nsIRequest* aRequest,
nsresult aStatusCode) {
MOZ_ASSERT(NS_IsMainThread());
auto clearBuf = MakeScopeExit([&] { mBuf.Truncate(); });
if (mAborted) {
return NS_OK;
}
bool success;
float confidence = 0;
Json::Value root;
Json::CharReaderBuilder builder;
bool parsingSuccessful;
nsAutoCString result;
nsAutoCString hypoValue;
nsAutoCString errorMsg;
SpeechRecognitionErrorCode errorCode;
SR_LOG("STT Result: %s", mBuf.get());
if (NS_FAILED(aStatusCode)) {
success = false;
errorMsg.AssignLiteral("Error connecting to the service.");
errorCode = SpeechRecognitionErrorCode::Network;
} else {
success = true;
UniquePtr<Json::CharReader> const reader(builder.newCharReader());
parsingSuccessful =
reader->parse(mBuf.BeginReading(), mBuf.EndReading(), &root, nullptr);
if (!parsingSuccessful) {
// there's an internal server error
success = false;
errorMsg.AssignLiteral("Internal server error");
errorCode = SpeechRecognitionErrorCode::Network;
} else {
result.Assign(root.get("status", "error").asString().c_str());
if (result.EqualsLiteral("ok")) {
// ok, we have a result
if (!root["data"].empty()) {
hypoValue.Assign(root["data"][0].get("text", "").asString().c_str());
confidence = root["data"][0].get("confidence", "0").asFloat();
} else {
success = false;
errorMsg.AssignLiteral("Error reading result data.");
errorCode = SpeechRecognitionErrorCode::Network;
}
} else {
success = false;
errorMsg.Assign(root.get("message", "").asString().c_str());
errorCode = SpeechRecognitionErrorCode::No_speech;
}
}
}
if (!success) {
mRecognition->DispatchError(
SpeechRecognition::EVENT_RECOGNITIONSERVICE_ERROR, errorCode, errorMsg);
} else {
// Declare javascript result events
RefPtr<SpeechEvent> event = new SpeechEvent(
mRecognition, SpeechRecognition::EVENT_RECOGNITIONSERVICE_FINAL_RESULT);
SpeechRecognitionResultList* resultList =
new SpeechRecognitionResultList(mRecognition);
SpeechRecognitionResult* result = new SpeechRecognitionResult(mRecognition);
if (mRecognition->MaxAlternatives() > 0) {
SpeechRecognitionAlternative* alternative =
new SpeechRecognitionAlternative(mRecognition);
alternative->mTranscript = NS_ConvertUTF8toUTF16(hypoValue);
alternative->mConfidence = confidence;
result->mItems.AppendElement(alternative);
}
resultList->mItems.AppendElement(result);
event->mRecognitionResultList = resultList;
NS_DispatchToMainThread(event);
}
return NS_OK;
}
OnlineSpeechRecognitionService::OnlineSpeechRecognitionService() = default;
OnlineSpeechRecognitionService::~OnlineSpeechRecognitionService() = default;
NS_IMETHODIMP
OnlineSpeechRecognitionService::Initialize(
WeakPtr<SpeechRecognition> aSpeechRecognition) {
MOZ_ASSERT(NS_IsMainThread());
mWriter = MakeUnique<OggWriter>();
mRecognition = new nsMainThreadPtrHolder<SpeechRecognition>(
"OnlineSpeechRecognitionService::mRecognition", aSpeechRecognition);
mEncodeTaskQueue = mRecognition->GetTaskQueueForEncoding();
MOZ_ASSERT(mEncodeTaskQueue);
return NS_OK;
}
void OnlineSpeechRecognitionService::EncoderFinished() {
MOZ_ASSERT(!NS_IsMainThread());
MOZ_ASSERT(mEncodedAudioQueue.IsFinished());
while (RefPtr<EncodedFrame> frame = mEncodedAudioQueue.PopFront()) {
AutoTArray<RefPtr<EncodedFrame>, 1> frames({frame});
DebugOnly<nsresult> rv =
mWriter->WriteEncodedTrack(frames, mEncodedAudioQueue.AtEndOfStream()
? ContainerWriter::END_OF_STREAM
: 0);
MOZ_ASSERT(NS_SUCCEEDED(rv));
}
mWriter->GetContainerData(&mEncodedData, ContainerWriter::FLUSH_NEEDED);
MOZ_ASSERT(mWriter->IsWritingComplete());
NS_DispatchToMainThread(
NewRunnableMethod("OnlineSpeechRecognitionService::DoSTT", this,
&OnlineSpeechRecognitionService::DoSTT));
}
void OnlineSpeechRecognitionService::EncoderInitialized() {
MOZ_ASSERT(!NS_IsMainThread());
AutoTArray<RefPtr<TrackMetadataBase>, 1> metadata;
metadata.AppendElement(mAudioEncoder->GetMetadata());
if (metadata[0]->GetKind() != TrackMetadataBase::METADATA_OPUS) {
SR_LOG("wrong meta data type!");
MOZ_ASSERT_UNREACHABLE();
}
nsresult rv = mWriter->SetMetadata(metadata);
MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
rv = mWriter->GetContainerData(&mEncodedData, ContainerWriter::GET_HEADER);
MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
Unused << rv;
}
void OnlineSpeechRecognitionService::EncoderError() {
MOZ_ASSERT(!NS_IsMainThread());
SR_LOG("Error encoding frames.");
mEncodedData.Clear();
NS_DispatchToMainThread(NS_NewRunnableFunction(
"SpeechRecognition::DispatchError",
[this, self = RefPtr<OnlineSpeechRecognitionService>(this)]() {
if (!mRecognition) {
return;
}
mRecognition->DispatchError(
SpeechRecognition::EVENT_RECOGNITIONSERVICE_ERROR,
SpeechRecognitionErrorCode::Audio_capture, "Encoder error");
}));
}
NS_IMETHODIMP
OnlineSpeechRecognitionService::ProcessAudioSegment(AudioSegment* aAudioSegment,
int32_t aSampleRate) {
MOZ_ASSERT(!NS_IsMainThread());
int64_t duration = aAudioSegment->GetDuration();
if (duration <= 0) {
return NS_OK;
}
if (!mAudioEncoder) {
mSpeechEncoderListener = new SpeechEncoderListener(this);
mAudioEncoder =
MakeUnique<OpusTrackEncoder>(aSampleRate, mEncodedAudioQueue);
RefPtr<AbstractThread> mEncoderThread = AbstractThread::GetCurrent();
mAudioEncoder->SetWorkerThread(mEncoderThread);
mAudioEncoder->RegisterListener(mSpeechEncoderListener);
}
mAudioEncoder->AppendAudioSegment(std::move(*aAudioSegment));
TimeStamp now = TimeStamp::Now();
if (mFirstIteration.IsNull()) {
mFirstIteration = now;
}
if ((now - mFirstIteration).ToMilliseconds() >= MAX_LISTENING_TIME_MS) {
NS_DispatchToMainThread(NS_NewRunnableFunction(
"SpeechRecognition::Stop",
[this, self = RefPtr<OnlineSpeechRecognitionService>(this)]() {
if (!mRecognition) {
return;
}
mRecognition->Stop();
}));
return NS_OK;
}
return NS_OK;
}
void OnlineSpeechRecognitionService::DoSTT() {
MOZ_ASSERT(NS_IsMainThread());
if (mAborted) {
return;
}
nsresult rv;
nsCOMPtr<nsIChannel> chan;
nsCOMPtr<nsIURI> uri;
nsAutoCString speechRecognitionEndpoint;
nsAutoCString prefEndpoint;
nsAutoString language;
Preferences::GetCString(PREFERENCE_DEFAULT_RECOGNITION_ENDPOINT,
prefEndpoint);
if (!prefEndpoint.IsEmpty()) {
speechRecognitionEndpoint = prefEndpoint;
} else {
speechRecognitionEndpoint = DEFAULT_RECOGNITION_ENDPOINT;
}
rv = NS_NewURI(getter_AddRefs(uri), speechRecognitionEndpoint, nullptr,
nullptr);
if (NS_WARN_IF(NS_FAILED(rv))) {
mRecognition->DispatchError(
SpeechRecognition::EVENT_RECOGNITIONSERVICE_ERROR,
SpeechRecognitionErrorCode::Network, "Unknown URI");
return;
}
nsSecurityFlags secFlags = nsILoadInfo::SEC_REQUIRE_CORS_INHERITS_SEC_CONTEXT;
nsLoadFlags loadFlags =
nsIRequest::LOAD_NORMAL | nsIChannel::LOAD_BYPASS_SERVICE_WORKER;
nsContentPolicyType contentPolicy = nsIContentPolicy::TYPE_OTHER;
nsPIDOMWindowInner* window = mRecognition->GetOwner();
if (NS_WARN_IF(!window)) {
mRecognition->DispatchError(
SpeechRecognition::EVENT_RECOGNITIONSERVICE_ERROR,
SpeechRecognitionErrorCode::Aborted, "No window");
return;
}
Document* doc = window->GetExtantDoc();
if (NS_WARN_IF(!doc)) {
mRecognition->DispatchError(
SpeechRecognition::EVENT_RECOGNITIONSERVICE_ERROR,
SpeechRecognitionErrorCode::Aborted, "No document");
}
rv = NS_NewChannel(getter_AddRefs(chan), uri, doc->NodePrincipal(), secFlags,
contentPolicy, nullptr, nullptr, nullptr, nullptr,
loadFlags);
if (NS_WARN_IF(NS_FAILED(rv))) {
mRecognition->DispatchError(
SpeechRecognition::EVENT_RECOGNITIONSERVICE_ERROR,
SpeechRecognitionErrorCode::Network, "Failed to open channel");
return;
}
nsCOMPtr<nsIHttpChannel> httpChan = do_QueryInterface(chan);
if (httpChan) {
rv = httpChan->SetRequestMethod("POST"_ns);
MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv));
}
if (httpChan) {
mRecognition->GetLang(language);
// Accept-Language-STT is a custom header of our backend server used to set
// the language of the speech sample being submitted by the client
rv = httpChan->SetRequestHeader("Accept-Language-STT"_ns,
NS_ConvertUTF16toUTF8(language), false);
MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv));
// Tell the server to not store the transcription by default
rv = httpChan->SetRequestHeader("Store-Transcription"_ns, "0"_ns, false);
MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv));
// Tell the server to not store the sample by default
rv = httpChan->SetRequestHeader("Store-Sample"_ns, "0"_ns, false);
MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv));
// Set the product tag as teh web speech api
rv = httpChan->SetRequestHeader("Product-Tag"_ns, "wsa"_ns, false);
MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv));
}
nsCOMPtr<nsIClassOfService> cos(do_QueryInterface(chan));
if (cos) {
cos->AddClassFlags(nsIClassOfService::UrgentStart);
}
nsCOMPtr<nsIUploadChannel2> uploadChan = do_QueryInterface(chan);
if (uploadChan) {
nsCOMPtr<nsIInputStream> bodyStream;
uint32_t length = 0;
for (const nsTArray<uint8_t>& chunk : mEncodedData) {
length += chunk.Length();
}
nsTArray<uint8_t> audio;
if (!audio.SetCapacity(length, fallible)) {
mRecognition->DispatchError(
SpeechRecognition::EVENT_RECOGNITIONSERVICE_ERROR,
SpeechRecognitionErrorCode::Audio_capture, "Allocation error");
return;
}
for (const nsTArray<uint8_t>& chunk : mEncodedData) {
audio.AppendElements(chunk);
}
mEncodedData.Clear();
rv = NS_NewByteInputStream(getter_AddRefs(bodyStream), std::move(audio));
if (NS_WARN_IF(NS_FAILED(rv))) {
mRecognition->DispatchError(
SpeechRecognition::EVENT_RECOGNITIONSERVICE_ERROR,
SpeechRecognitionErrorCode::Network, "Failed to open stream");
return;
}
if (bodyStream) {
rv = uploadChan->ExplicitSetUploadStream(bodyStream, "audio/ogg"_ns,
length, "POST"_ns, false);
MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv));
}
}
rv = chan->AsyncOpen(this);
if (NS_WARN_IF(NS_FAILED(rv))) {
mRecognition->DispatchError(
SpeechRecognition::EVENT_RECOGNITIONSERVICE_ERROR,
SpeechRecognitionErrorCode::Network, "Internal server error");
}
}
NS_IMETHODIMP
OnlineSpeechRecognitionService::SoundEnd() {
MOZ_ASSERT(NS_IsMainThread());
if (!mEncodeTaskQueue) {
// Not initialized
return NS_OK;
}
nsresult rv = mEncodeTaskQueue->Dispatch(NS_NewRunnableFunction(
"OnlineSpeechRecognitionService::SoundEnd",
[this, self = RefPtr<OnlineSpeechRecognitionService>(this)]() {
if (mAudioEncoder) {
mAudioEncoder->NotifyEndOfStream();
mAudioEncoder->UnregisterListener(mSpeechEncoderListener);
mSpeechEncoderListener = nullptr;
mAudioEncoder = nullptr;
EncoderFinished();
}
}));
MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
Unused << rv;
mEncodeTaskQueue = nullptr;
return NS_OK;
}
NS_IMETHODIMP
OnlineSpeechRecognitionService::ValidateAndSetGrammarList(
SpeechGrammar* aSpeechGrammar,
nsISpeechGrammarCompilationCallback* aCallback) {
// This is an online LVCSR (STT) service,
// so we don't need to set a grammar
return NS_OK;
}
NS_IMETHODIMP
OnlineSpeechRecognitionService::Abort() {
MOZ_ASSERT(NS_IsMainThread());
if (mAborted) {
return NS_OK;
}
mAborted = true;
return SoundEnd();
}
} // namespace mozilla