Class VoiceMLModule

Allows the Lens to incorporate transcription, keyword detection, voice command detection and other NLP based features into Lenses.

See

VoiceML guide.

Example

//@input Asset.VoiceMLModule vmlModule {"label": "Voice ML Module"}
//@input Asset.AudioTrackAsset audioTrack

var options = VoiceML.ListeningOptions.create();
options.speechRecognizer = VoiceMLModule.SpeechRecognizer.Default;

//General Option
options.shouldReturnAsrTranscription = true;
options.shouldReturnInterimAsrTranscription = true;

//Speech Context
var phrasesOne = ["carrot", "tomato"];
var boostValueOne = 5;
options.addSpeechContext(phrasesOne,boostValueOne);

var phrasesTwo = ["orange", "apple"];
var boostValueTwo = 6;
options.addSpeechContext(phrasesTwo,boostValueTwo);


//NLPKeywordModel
var nlpKeywordModel = VoiceML.NlpKeywordModelOptions.create(); 
nlpKeywordModel.addKeywordGroup("Vegetable", ["carrot", "tomato"]);
nlpKeywordModel.addKeywordGroup("Fruit", ["orange", "apple"]);

//Command
var nlpIntentModel = VoiceML.NlpIntentsModelOptions.create("VOICE_ENABLED_UI");       
nlpIntentModel.possibleIntents =  ["next", "back", "left", "right", "up", "down", "first", "second", "third", "fourth", "fifth", "sixth", "seventh", "eighth", "ninth", "tenth"];
options.nlpModels =[nlpKeywordModel, nlpIntentModel];

var onListeningEnabledHandler = function() {
    script.vmlModule.startListening(options);
};
var onListeningDisabledHandler = function() {
    script.vmlModule.stopListening();
};

var getErrorMessage = function(response) {
    var errorMessage = "";
    switch (response) {
        case "#SNAP_ERROR_INDECISIVE":
            errorMessage = "indecisive";
            break;
        case "#SNAP_ERROR_INCONCLUSIVE":
            errorMessage = "inconclusive";
            break;
        case "#SNAP_ERROR_NONVERBAL":
            errorMessage = "non verbal";
            break;
        case "#SNAP_ERROR_SILENCE":
            errorMessage = "too long silence";
            break;          
        default:
        if (response.includes("#SNAP_ERROR")) {
            errorMessage = "general error";  
        } else {
            errorMessage = "unknown error";
        }
    }
    return errorMessage;
};

var parseKeywordResponses = function(keywordResponses) {
    var keywords = [];
    var code = "";
    for (var kIterator = 0; kIterator < keywordResponses.length; kIterator++) {
        var keywordResponse = keywordResponses[kIterator];
        
        switch (keywordResponse.status.code) {
            case VoiceMLModule.NlpResponsesStatusCodes.OK:
                code= "OK";
                for (var keywordsIterator = 0; keywordsIterator < keywordResponse.keywords.length; keywordsIterator++) {
                    var keyword = keywordResponse.keywords[keywordsIterator];
                    if (keyword.includes("#SNAP_ERROR")) {
                        var errorMessage = getErrorMessage(keyword);
                        print("Keyword Error: " + errorMessage);
                        break;
                    }
                    keywords.push(keyword);                  
                }
                break;
            case VoiceMLModule.NlpResponsesStatusCodes.ERROR:
                code = "ERROR";
                print("Status Code: "+code+ " Description: " + keywordResponse.status.code.description);
                break;
            default:
                print("Status Code: No Status Code");
        }
    }
    return keywords;
};

var parseCommandResponses = function(commandResponses) {
    var commands = [];
    var code = "";
    for (var iIterator = 0; iIterator < commandResponses.length; iIterator++) {
        var commandResponse = commandResponses[iIterator];
        switch (commandResponse.status.code) {
            case VoiceMLModule.NlpResponsesStatusCodes.OK:
                code= "OK";
                var command = commandResponse.intent;
                if (command.includes("#SNAP_ERROR")) {
                    var errorMessage = getErrorMessage(command);
                    print("Command Error: " + errorMessage);
                    break;
                }
                commands.push(commandResponse.intent);
                break;
            case VoiceMLModule.NlpResponsesStatusCodes.ERROR:
                code = "ERROR";
                print("Status Code: "+code+ " Description: " + commandResponse.status.code.description);
                break;
            default:
                print("Status Code: No Status Code");
        } 
    }
    return commands;
};

var onUpdateListeningEventHandler = function(eventArgs) { 
    if (eventArgs.transcription.trim() == "") {
        return;
    }  
    print("Transcription: " + eventArgs.transcription);
    
    if (!eventArgs.isFinalTranscription) {
        return;
    }  
    print("Final Transcription: " + eventArgs.transcription); 
    
    //Keyword Results 
    var keywordResponses = eventArgs.getKeywordResponses();  
    var keywords = parseKeywordResponses(keywordResponses);
    if (keywords.length > 0) {
        var keywordResponseText = "";
        for (var kIterator=0;kIterator<keywords.length;kIterator++) {
            keywordResponseText += keywords[kIterator] +"\n";                                   
        }
        print("Keywords:" + keywordResponseText);
    }
    
    //Command Results  
    var commandResponses = eventArgs.getIntentResponses();    
    var commands = parseCommandResponses(commandResponses);
    
    if (commands.length > 0) {
        var commandResponseText = ""; 
        for (var iIterator=0;iIterator<commands.length;iIterator++) {
            commandResponseText += commands[iIterator]+"\n";
        }   
        print("Commands: " + commandResponseText);
    }
    
};

var onListeningErrorHandler = function(eventErrorArgs) {
    print("Error: " + eventErrorArgs.error + " desc: "+ eventErrorArgs.description);
};

//VoiceML Callbacks
script.vmlModule.onListeningUpdate.add(onUpdateListeningEventHandler);
script.vmlModule.onListeningError.add(onListeningErrorHandler); 
script.vmlModule.onListeningEnabled.add(onListeningEnabledHandler);
script.vmlModule.onListeningDisabled.add(onListeningDisabledHandler);

Hierarchy (View Summary, Expand)

Constructors

constructor

Properties

name onListeningDisabled onListeningEnabled onListeningError onListeningUpdate uniqueIdentifier

Methods

enableSystemCommands getTypeName isOfType isSame startListening stopListening

Constructors

Protectedconstructor

Properties

name

name: string

The name of the Asset in Lens Studio.

ReadonlyonListeningDisabled

onListeningDisabled: event0<void>

Registers a callback which will be called when microphone permissions are taken from the lens. stopListening() is implicitly called in such case.

ReadonlyonListeningEnabled

onListeningEnabled: event0<void>

Registers a callback which will be called when microphone permissions are granted to the Lens, the microphone is initialized, and is actively listening. The expected design pattern is to start the listening session once those permissions have been granted:

//@input Asset.VoiceMLModule vmlModule
var onListeningEnabledHandler = function(){
    script.vmlModule.startListening(options);  
}
script.vmlModule.onListeningEnabled.add(onListeningEnabledHandler);

ReadonlyonListeningError

onListeningError: event1<ListeningErrorEventArgs, void>

Registers a callback, which will be called in case the VoiceML module can't process the inputs. Most errors are due to network connectivity, or misconfigured NLP inputs.

ReadonlyonListeningUpdate

onListeningUpdate: event1<ListeningUpdateEventArgs, void>

Registers a callback, which will be called with interim transcription or related NLP models.

ReadonlyuniqueIdentifier

uniqueIdentifier: string

Methods

enableSystemCommands

  • enableSystemCommands(): void

    Allows the user to provide voice commands for the VoiceML to execute on behalf of the users. Current supported commands: "Take a Snap", "Start Recording", "Stop Recording". In case a command was detected, it will be automtically executed by the system and returned as part of the VoiceML.NlpCommandResponse in the onListeningUpdate callback. You can retrieve the command that was executed using the following snippet:

    var onUpdateListeningEventHandler = function(eventArgs) {
        var commandResponses = eventArgs.getCommandResponses();
        var nlpResponseText = "";

        for (var i = 0; i < commandResponses.length; i++) {
            var commandResponse = commandResponses[i];
            nlpResponseText += "Command Response: " + commandResponse.modelName + "\n command: " + commandResponse.command;     
        }
    }

    Returns void

getTypeName

  • getTypeName(): string

    Returns the name of this object's type.

    Returns string

isOfType

  • isOfType(type: string): boolean

    Returns true if the object matches or derives from the passed in type.

    Parameters

    • type: string

    Returns boolean

isSame

  • isSame(other: ScriptObject): boolean

    Returns true if this object is the same as other. Useful for checking if two references point to the same thing.

    Parameters

    Returns boolean

startListening

  • startListening(options: ListeningOptions): void
    Exposes User Data

    Starts transcribing the user audio, the NLP model is connected and sends back results using an event, optionally could request for transcription and interim results. Notice, you can only startListening, after microphone permissions have been granted. It is recommended to use startListening method only after onListeningEnabled callback was called.

    Parameters

    Returns void

stopListening

  • stopListening(): void

    Stops voice commands.

    Returns void

Settings

Member Visibility

On This Page

Constructors
Properties
Methods
MMNEPVFCICPMFPCPTTAAATR