import { useEffect, useRef, useState } from "react"; import { SandaiClient, UrlBuilder, VoiceNames, Environments, AuthClientMessage, } from "sandai-core/index"; export type FocusProps = { /** * Controls the strength of the camera’s tracking behavior. * * Higher values typically result in faster or tighter camera movement * toward the target, while lower values produce smoother, looser motion. * * @defaultValue Implementation-defined (`0.005`) */ focusIntensity?: number; /** * Positional offset applied to the camera relative to its base position. * * This is commonly used to shift the camera up, back, or sideways * without changing the look-at target. * * @defaultValue `new Vector3(0, 0, 0)` */ cameraOffset?: { x: number; y: number; z: number }; /** * Offset applied to the camera’s look-at target. * * Useful for aiming the camera slightly above or to the side of * a character. * * @defaultValue `new Vector3(0, 0, 0)` */ lookAtOffset?: { x: number; y: number; z: number }; /** * Whether the camera should dynamically track the character’s look-at * direction instead of the head bones'. * * When enabled, the camera’s target may update in real time as the * character rotates or changes the orientation of the eyes. * * @defaultValue `false` */ trackCharacterLookAt?: boolean; /** * Sets the camera mode to focus the models head, so it moves * along with the head, or to be independent * * @defaultValie `focus` */ focusMode?: "focus" | "unfocus"; }; /** * Props for the `AI3DCharacter` component. * * @property {string} url - The base URL for the Sandai AI 3D character chat interface. * @property {(client: SandaiClient) => void} onLoad - Callback function triggered when the SandaiClient is loaded. * @property {string} [vrmUrl] - Optional URL for the VRM model to be loaded as the 3D character. * @property {VoiceNames} [voiceName] - Optional name of the voice to be used by the 3D character. * @property {boolean} [showControls] - Optional flag to determine if controls should be displayed in the interface. */ export type AI3DCharacterProps = { url: string; onLoad: (client: SandaiClient) => void; vrmUrl?: string; voiceName?: VoiceNames; style?: React.CSSProperties; className?: string; environment?: Environments; /** * Shows the Debug-Interface for the built-in engine, * giving you access to the predefined motions, * emotions and the built-in voices as a neat * overlay. * * Since this technology works with vrm models, * and since those models only have 5 emotions by default, * it can be useful for looking at how the emotions combine * to show the 27 emotions that sandai supports */ showControls?: boolean; /** * Catch-all kind of a switch that will make things run on * less powerful hardware. There are a few subtle effects * like bloom and depth of field that are enabled by default, * but they can be resource-intensive. On modern hardware, * they run fine, but if you need to support, say, older * android phones, then turning this on is a good idea. * * Some effects may also cause issues on outdated hardware * or software, so turning this on by default may also be * a good idea in general, but your milage may vary. */ lowPerformanceMode?: boolean; /** * The type of camera the scene should be rendered with * "default" is the standard perspective camera * "orthographic" is an orhtographic camera * * Since the two camera types work differently, things like * focus have different methods of calculation. * * For example, having a z-axis in an orthographic camera to * control it's position and therefore the relative size of * what's in view, the zoom is calculated from that value * instead. * * It still works with the out of box experience, but if you * set focus with custom values and switch the camera type, * your carefully crafted values will likely look a bit off. * * If this annoys you, you can open an issue and send a pull- * request in the r3f-vrm package in gitlab */ cameraType?: "default" | "orthographic"; /** * Dampens the head movements. * * This is a number between 0 and 1, where 0 is the default, * so the motions are applied as is, and 1 effectively disables * all bone movement. */ headAnimationSmoothing?: number; /** * Dampens the hand movements. * * This is a number between 0 and 1, where 0 is the default, * so the motions are applied as is, and 1 effectively disables * all bone movement. */ handsAnimationSmoothing?: number; /** * Dampens the legs movements. * * This is a number between 0 and 1, where 0 is the default, * so the motions are applied as is, and 1 effectively disables * all bone movement. */ legsAnimationSmoothing?: number; /** * Dampens the body movements. * * This is a number between 0 and 1, where 0 is the default, * so the motions are applied as is, and 1 effectively disables * all bone movement. */ bodyAnimationSmoothing?: number; /** * Dampens the arms movements. * * This is a number between 0 and 1, where 0 is the default, * so the motions are applied as is, and 1 effectively disables * all bone movement. */ armsAnimationSmoothing?: number; /** * Dampens the face movements. * * This is a number between 0 and 1, where 0 is the default, * so the motions are applied as is, and 1 effectively disables * all bone movement. * * Also handles face expression smoothing. * * 0 is the default and * only applies the default transitory smoothing, meaning for example * when the mouth is maxed out at "aa" for 2 seconds and then goes * to "ih", the transition is already smoothed out. * * 1 basically holds the vowel indefinitely. * * If the mouth movements feel too choppy, try setting this value * to somewhere between 0.1 and 0.9 */ faceAnimationSmoothing?: number; /** * The initial camera properties */ initialFocus?: FocusProps; /** * Your API key. You can find it in the dashboard */ apiKey?: AuthClientMessage["apiKey"]; /** * Your user ID. You can find it in the dashboard */ userId?: AuthClientMessage["userId"]; /** * Skips auth and load checks. * * Load checks are used to determine if the user has * interacted with the iframe yet (among other things), * so this should be turned off in prod. */ debugMode?: boolean; /** * Whether to use raytracing or not. */ raytrace?: boolean; }; /** * A React component that renders a 3D character using Sandai. * * This component dynamically constructs the URL based on the provided props and initializes * a `SandaiClient` instance to interact with the 3D character. * * @param {AI3DCharacterProps} props - The properties passed to the component. * @param {string} props.url - The base URL for the Sandai 3D character iframe. * @param {(client: SandaiClient) => void} props.onLoad - Callback function triggered when the SandaiClient is loaded. * @param {string} [props.vrmUrl] - Optional URL for the VRM model to be loaded in the 3D character. * @param {VoiceNames} [props.voiceName] - Optional name of the voice to be used by the 3D character. * @param {boolean} [props.showControls] - Optional flag to determine if controls should be displayed in the iframe. * @param {boolean} [props.lowPerformanceMode] - Optional flag to increase performance at the cost of looking nice. * @param {"default" | "orthographic"} [props.cameraType] - Sets the camera to either default (which is the Perspective Camera) or orthographic * @param {number} [props.headAnimationSmoothing] - Dampens head movements. Value between 0 and 1, where 0 is no dampening and 1 effectively disables movements * @param {number} [props.bodyAnimationSmoothing] - Dampens body movements. Value between 0 and 1, where 0 is no dampening and 1 effectively disables movements * @param {number} [props.armsAnimationSmoothing] - Dampens arms movements. Value between 0 and 1, where 0 is no dampening and 1 effectively disables movements * @param {number} [props.legsAnimationSmoothing] - Dampens legs movements. Value between 0 and 1, where 0 is no dampening and 1 effectively disables movements * @param {number} [props.handsAnimationSmoothing] - Dampens hands movements. Value between 0 and 1, where 0 is no dampening and 1 effectively disables movements * @param {number} [props.faceAnimationSmoothing] - Dampens face movements. Value between 0 and 1, where 0 is no dampening and 1 effectively disables movements * @param {FocusProps} [props.initialFocus] - The initial camera options like if the model should be focused or where the camera should be * @param {AuthClientMessage["apiKey"]} [props.apiKey] - API key, available in the sandai dashboard * @param {AuthClientMessage["userId"]} [props.userId] - userId, available in the sandai dashboard * @param {boolean} [props.debugMode] - For when you're developing and auth and load state can be ignored. This will skip the interaction * @param {boolean} [props.raytrace] - Whether to use Raytracing or not. * @returns {JSX.Element} - Returns an iframe element that displays the 3D character. */ export const AI3DCharacter = (props: AI3DCharacterProps) => { const clientRef = useRef(); // State for the iframe URL const [iframeSrc, setIframeSrc] = useState(""); // useEffect to build the URL and set the iframe src when parameters change useEffect(() => { const builtUrl = new UrlBuilder(props.url) .setParam("vrmUrl", props.vrmUrl) .setParam("voiceName", props.voiceName) .setParam("showControls", props.showControls) .setParam("environment", props.environment) .setParam("lowPerformanceMode", props.lowPerformanceMode) .setParam("cameraType", props.cameraType) .setParam("bodyAnimationSmoothing", props.bodyAnimationSmoothing) .setParam("headAnimationSmoothing", props.headAnimationSmoothing) .setParam("armsAnimationSmoothing", props.armsAnimationSmoothing) .setParam("handsAnimationSmoothing", props.handsAnimationSmoothing) .setParam("legsAnimationSmoothing", props.legsAnimationSmoothing) .setParam("faceAnimationSmoothing", props.faceAnimationSmoothing) .setParam("initialFocus", props.initialFocus) .setParam("debugMode", props.debugMode) .setParam("raytrace", props.raytrace) .build(); setIframeSrc(builtUrl); }, [ props.vrmUrl, props.voiceName, props.showControls, props.url, props.environment, props.lowPerformanceMode, ]); useEffect(() => { if (!iframeSrc) return; if (!clientRef.current) { clientRef.current = new SandaiClient( "sandai-frame", props.userId, props.apiKey, { skipAuth: props.debugMode ?? false, skipLoadCheck: props.debugMode ?? false, }, ); } clientRef.current.init().then(() => { props.onLoad(clientRef.current!); }); return () => { clientRef.current?.destroy(); clientRef.current = undefined; }; }, [props.url, props.onLoad, iframeSrc]); if (iframeSrc === "") return; return ( ); };