Lens Scripting API
    Preparing search index...

    Class VoiceMLModule

    Allows the Lens to incorporate transcription, keyword detection, voice command detection and other NLP based features into Lenses.

    VoiceML guide.

    Example usage of the VoiceMLModule to configure listening options, apply speech contexts, handle keyword and intent models, process transcription responses, and handle listening events/errors.

    //@input Asset.VoiceMLModule vmlModule {"label": "Voice ML Module"}
    //@input Asset.AudioTrackAsset audioTrack

    var options = VoiceML.ListeningOptions.create();
    options.speechRecognizer = VoiceMLModule.SpeechRecognizer.Default;

    // General Options
    options.shouldReturnAsrTranscription = true;
    options.shouldReturnInterimAsrTranscription = true;

    // Speech Context
    var phrasesOne = ["carrot", "tomato"];
    var boostValueOne = 5;
    options.addSpeechContext(phrasesOne, boostValueOne);

    var phrasesTwo = ["orange", "apple"];
    var boostValueTwo = 6;
    options.addSpeechContext(phrasesTwo, boostValueTwo);

    // NLP Keyword Model
    var nlpKeywordModel = VoiceML.NlpKeywordModelOptions.create();
    nlpKeywordModel.addKeywordGroup("Vegetable", ["carrot", "tomato"]);
    nlpKeywordModel.addKeywordGroup("Fruit", ["orange", "apple"]);

    // NLP Intent Model
    var nlpIntentModel = VoiceML.NlpIntentsModelOptions.create("VOICE_ENABLED_UI");
    nlpIntentModel.possibleIntents = [
      "next", "back", "left", "right", "up", "down",
      "first", "second", "third", "fourth", "fifth",
      "sixth", "seventh", "eighth", "ninth", "tenth"
    ];
    options.nlpModels = [nlpKeywordModel, nlpIntentModel];

    var onListeningEnabledHandler = function() {
        script.vmlModule.startListening(options);
    };
    var onListeningDisabledHandler = function() {
        script.vmlModule.stopListening();
    };

    var getErrorMessage = function(response) {
        var errorMessage = "";
        switch (response) {
            case "#SNAP_ERROR_INDECISIVE": errorMessage = "indecisive"; break;
            case "#SNAP_ERROR_INCONCLUSIVE": errorMessage = "inconclusive"; break;
            case "#SNAP_ERROR_NONVERBAL": errorMessage = "non verbal"; break;
            case "#SNAP_ERROR_SILENCE": errorMessage = "too long silence"; break;
            default:
                if (response.includes("#SNAP_ERROR")) {
                    errorMessage = "general error";
                } else {
                    errorMessage = "unknown error";
                }
        }
        return errorMessage;
    };

    var parseKeywordResponses = function(keywordResponses) {
        var keywords = [];
        var code = "";
        for (var kIterator = 0; kIterator < keywordResponses.length; kIterator++) {
            var keywordResponse = keywordResponses[kIterator];
            switch (keywordResponse.status.code) {
                case VoiceMLModule.NlpResponsesStatusCodes.OK:
                    code = "OK";
                    for (var keywordsIterator = 0; keywordsIterator < keywordResponse.keywords.length; keywordsIterator++) {
                        var keyword = keywordResponse.keywords[keywordsIterator];
                        if (keyword.includes("#SNAP_ERROR")) {
                            var errorMessage = getErrorMessage(keyword);
                            print("Keyword Error: " + errorMessage);
                            break;
                        }
                        keywords.push(keyword);
                    }
                    break;
                case VoiceMLModule.NlpResponsesStatusCodes.ERROR:
                    code = "ERROR";
                    print("Status Code: " + code + " Description: " + keywordResponse.status.code.description);
                    break;
                default:
                    print("Status Code: No Status Code");
            }
        }
        return keywords;
    };

    var parseCommandResponses = function(commandResponses) {
        var commands = [];
        var code = "";
        for (var iIterator = 0; iIterator < commandResponses.length; iIterator++) {
            var commandResponse = commandResponses[iIterator];
            switch (commandResponse.status.code) {
                case VoiceMLModule.NlpResponsesStatusCodes.OK:
                    code = "OK";
                    var command = commandResponse.intent;
                    if (command.includes("#SNAP_ERROR")) {
                        var errorMessage = getErrorMessage(command);
                        print("Command Error: " + errorMessage);
                        break;
                    }
                    commands.push(commandResponse.intent);
                    break;
                case VoiceMLModule.NlpResponsesStatusCodes.ERROR:
                    code = "ERROR";
                    print("Status Code: " + code + " Description: " + commandResponse.status.code.description);
                    break;
                default:
                    print("Status Code: No Status Code");
            }
        }
        return commands;
    };

    var onUpdateListeningEventHandler = function(eventArgs) {
        if (eventArgs.transcription.trim() == "") {
            return;
        }
        print("Transcription: " + eventArgs.transcription);

        if (!eventArgs.isFinalTranscription) {
            return;
        }
        print("Final Transcription: " + eventArgs.transcription);

        // Keyword Results
        var keywordResponses = eventArgs.getKeywordResponses();
        var keywords = parseKeywordResponses(keywordResponses);
        if (keywords.length > 0) {
            var keywordResponseText = "";
            for (var kIterator = 0; kIterator < keywords.length; kIterator++) {
                keywordResponseText += keywords[kIterator] + "\n";
            }
            print("Keywords:" + keywordResponseText);
        }

        // Command Results
        var commandResponses = eventArgs.getIntentResponses();
        var commands = parseCommandResponses(commandResponses);
        if (commands.length > 0) {
            var commandResponseText = "";
            for (var iIterator = 0; iIterator < commands.length; iIterator++) {
                commandResponseText += commands[iIterator] + "\n";
            }
            print("Commands: " + commandResponseText);
        }
    };

    var onListeningErrorHandler = function(eventErrorArgs) {
        print("Error: " + eventErrorArgs.error + " desc: " + eventErrorArgs.description);
    };

    // VoiceML Callbacks
    script.vmlModule.onListeningUpdate.add(onUpdateListeningEventHandler);
    script.vmlModule.onListeningError.add(onListeningErrorHandler);
    script.vmlModule.onListeningEnabled.add(onListeningEnabledHandler);
    script.vmlModule.onListeningDisabled.add(onListeningDisabledHandler);

    Hierarchy (View Summary)

    Index

    Properties

    name onListeningDisabled onListeningEnabled onListeningError onListeningUpdate uniqueIdentifier

    Methods

    enableSystemCommands getTypeName isOfType isSame startListening stopListening

    Properties

    name: string

    The name of the Asset in Lens Studio.

    onListeningDisabled: event0<void>

    Registers a callback which will be called when microphone permissions are taken from the lens. stopListening() is implicitly called in such case.

    onListeningEnabled: event0<void>

    Registers a callback which will be called when microphone permissions are granted to the Lens, the microphone is initialized, and is actively listening. The expected design pattern is to start the listening session once those permissions have been granted:

    //@input Asset.VoiceMLModule vmlModule
    var onListeningEnabledHandler = function(){
        script.vmlModule.startListening(options);
    }
    script.vmlModule.onListeningEnabled.add(onListeningEnabledHandler);
    onListeningError: event1<ListeningErrorEventArgs, void>

    Registers a callback, which will be called in case the VoiceML module can't process the inputs. Most errors are due to network connectivity, or misconfigured NLP inputs.

    onListeningUpdate: event1<ListeningUpdateEventArgs, void>

    Registers a callback, which will be called with interim transcription or related NLP models.

    uniqueIdentifier: string

    Methods

    • Allows the user to provide voice commands for the VoiceML to execute on behalf of the users. Current supported commands: "Take a Snap", "Start Recording", "Stop Recording". In case a command was detected, it will be automtically executed by the system and returned as part of the VoiceML.NlpCommandResponse in the onListeningUpdate callback. You can retrieve the command that was executed using the following snippet:

      var onUpdateListeningEventHandler = function(eventArgs) {
          var commandResponses = eventArgs.getCommandResponses();
          var nlpResponseText = "";

          for (var i = 0; i < commandResponses.length; i++) {
              var commandResponse = commandResponses[i];
              nlpResponseText += "Command Response: " + commandResponse.modelName + "\n command: " + commandResponse.command;
          }
      }

      Returns void

    • Returns the name of this object's type.

      Returns string

    • Returns true if the object matches or derives from the passed in type.

      Parameters

      • type: string

      Returns boolean

    • Returns true if this object is the same as other. Useful for checking if two references point to the same thing.

      Parameters

      Returns boolean

    • Exposes User Data

      Starts transcribing the user audio, the NLP model is connected and sends back results using an event, optionally could request for transcription and interim results. Notice, you can only startListening, after microphone permissions have been granted. It is recommended to use startListening method only after onListeningEnabled callback was called.

      Parameters

      Returns void

    • Stops voice commands.

      Returns void

    Settings

    Member Visibility

    On This Page

    Properties
    Methods