Hand Tracking

The Spectacles Interaction Kit enables developers to create immersive augmented reality experiences using hand tracking as an input method. This technology allows users to interact with their environment in a natural way, enhancing the sense of presence and social interaction. By leveraging hand tracking, developers can create more dynamic and realistic AR interactions.

Developers can utilize several hand interaction modes within the Spectacles Interaction Kit, including indirect (ray-based) interactions, direct-pinch, and direct-poke, to enhance their lenses. These interaction modalities offer flexibility in user engagement with AR content, allowing for contextually appropriate experiences. SIK’s hand tracking input also works in conjunction with other input methods, such as mobile controller input, providing a multi-faceted approach to user interaction with lenses.

Relevant Components

Code Example - Hand Input Data

HandInputData interfaces with Lens Studio’s ObjectTracking3D and GestureModule to provide hand tracking data to the rest of SIK, and in particular to the HandInteractor and HandVisual components. Please note that HandInputData provides hand tracking data only. TrackedHand represents and manages a tracked hand. It can be accessed using HandInputData’s getHand API. TrackedHand provides useful properties, including all keypoints of the hand, among others.

TypeScript

import { SIK } from './SpectaclesInteractionKit/SIK';

@component
export class ExampleHandScript extends BaseScriptComponent {
  onAwake() {
    this.createEvent('OnStartEvent').bind(() => {
      this.onStart();
    });
  }

  onStart() {
    // Retrieve HandInputData from SIK's definitions.
    let handInputData = SIK.HandInputData;

    // Fetch the TrackedHand for left and right hands.
    let leftHand = handInputData.getHand('left');
    let rightHand = handInputData.getHand('right');

    // Add print callbacks for whenever these hands pinch.
    leftHand.onPinchDown.add(() => {
      print(
        `The left hand has pinched. The tip of the left index finger is: ${leftHand.indexTip.position}.`
      );
    });
    rightHand.onPinchDown.add(() => {
      print(
        `The right hand has pinched. The tip of the right index finger is: ${rightHand.indexTip.position}.`
      );
    });
  }
}

JavaScript

const SIK = require('SpectaclesInteractionKit/SIK').SIK;
const handInputData = SIK.HandInputData;

function onAwake() {
  // Wait for other components to initialize by deferring to OnStartEvent.
  script.createEvent('OnStartEvent').bind(() => {
    onStart();
  });
}

function onStart() {
  // Fetch the TrackedHand for left and right hands.
  var leftHand = handInputData.getHand('left');
  var rightHand = handInputData.getHand('right');

  // Add print callbacks for whenever these hands pinch.
  leftHand.onPinchDown.add(() => {
    print(
      `The left hand has pinched. The tip of the left index finger is: ${leftHand.indexTip.position}.`
    );
  });
  rightHand.onPinchDown.add(() => {
    print(
      `The right hand has pinched. The tip of the right index finger is: ${rightHand.indexTip.position}.`
    );
  });
}

onAwake();

Code Example - HandInteractor

HandInteractor processes information from HandInputData to interact with scene objects in the lens that have an Interactable component. For more information about the interaction system, refer to Interaction System.

TypeScript

import { SIK } from './SpectaclesInteractionKit/SIK';
import { InteractorInputType } from './SpectaclesInteractionKit/Core/Interactor/Interactor';
@component
export class ExampleHandScript extends BaseScriptComponent {
  onAwake() {
    this.createEvent('UpdateEvent').bind(() => {
      this.onUpdate();
    });
  }

  onUpdate() {
    // Retrieve InteractionManager from SIK's definitions.
    let interactionManager = SIK.InteractionManager;

    // Fetch the HandInteractor for left and right hands.
    let leftHandInteractor = interactionManager.getInteractorsByType(
      InteractorInputType.LeftHand
    )[0];
    let rightHandInteractor = interactionManager.getInteractorsByType(
      InteractorInputType.RightHand
    )[0];

    // Print the position and direction of the HandInteractors each frame.
    print(
      `The left hand interactor is at position: ${leftHandInteractor.startPoint} and is pointing in direction: ${leftHandInteractor.direction}.`
    );
    print(
      `The right hand interactor is at position: ${rightHandInteractor.startPoint} and is pointing in direction: ${rightHandInteractor.direction}.`
    );
  }
}

JavaScript

const SIK = require('SpectaclesInteractionKit/SIK').SIK;
const interactionManager = SIK.InteractionManager;
const interactorInputType =
  require('SpectaclesInteractionKit/Core/Interactor/Interactor').InteractorInputType;

function onAwake() {
  // Wait for other components to initialize by deferring to OnStartEvent.
  script.createEvent('UpdateEvent').bind(() => {
    onUpdate();
  });
}

function onUpdate() {
  // Fetch the HandInteractor for left and right hands.
  var leftHandInteractor = interactionManager.getInteractorsByType(
    interactorInputType.LeftHand
  )[0];
  var rightHandInteractor = interactionManager.getInteractorsByType(
    interactorInputType.RightHand
  )[0];

  // Print the position and direction of the HandInteractors each frame.
  print(
    `The left hand interactor is at position: ${leftHandInteractor.startPoint} and is pointing in direction: ${leftHandInteractor.direction}.`
  );
  print(
    `The right hand interactor is at position: ${rightHandInteractor.startPoint} and is pointing in direction: ${rightHandInteractor.direction}.`
  );
}

onAwake();