{"version":3,"file":"av-cliper.umd.cjs","sources":["../src/chromakey.ts","../src/dom-utils.ts","../src/av-utils.ts","../src/clips/iclip.ts","../src/mp4-utils/mp4box-utils.ts","../src/clips/mp4-clip.ts","../src/clips/img-clip.ts","../src/clips/audio-clip.ts","../src/clips/media-stream-clip.ts","../src/clips/embed-subtitles-clip.ts","../src/mp4-utils/sample-transform.ts","../src/mp4-utils/index.ts","../src/combinator.ts","../src/sprite/rect.ts","../src/sprite/base-sprite.ts","../src/sprite/offscreen-sprite.ts","../src/sprite/visible-sprite.ts"],"sourcesContent":["// 改编自 https://jameshfisher.com/2020/08/11/production-ready-green-screen-in-the-browser/\r\nconst vertexShader = `#version 300 es\r\n layout (location = 0) in vec4 a_position;\r\n layout (location = 1) in vec2 a_texCoord;\r\n out vec2 v_texCoord;\r\n void main () {\r\n gl_Position = a_position;\r\n v_texCoord = a_texCoord;\r\n }\r\n`;\r\n\r\nconst fragmentShader = `#version 300 es\r\nprecision mediump float;\r\nout vec4 FragColor;\r\nin vec2 v_texCoord;\r\n\r\nuniform sampler2D frameTexture;\r\nuniform vec3 keyColor;\r\n\r\n// 色度的相似度计算\r\nuniform float similarity;\r\n// 透明度的平滑度计算\r\nuniform float smoothness;\r\n// 降低绿幕饱和度，提高抠图准确度\r\nuniform float spill;\r\n\r\nvec2 RGBtoUV(vec3 rgb) {\r\n return vec2(\r\n rgb.r * -0.169 + rgb.g * -0.331 + rgb.b * 0.5 + 0.5,\r\n rgb.r * 0.5 + rgb.g * -0.419 + rgb.b * -0.081 + 0.5\r\n );\r\n}\r\n\r\nvoid main() {\r\n // 获取当前像素的rgba值\r\n vec4 rgba = texture(frameTexture, v_texCoord);\r\n // 计算当前像素与绿幕像素的色度差值\r\n vec2 chromaVec = RGBtoUV(rgba.rgb) - RGBtoUV(keyColor);\r\n // 计算当前像素与绿幕像素的色度距离（向量长度）, 越相像则色度距离越小\r\n float chromaDist = sqrt(dot(chromaVec, chromaVec));\r\n // 设置了一个相似度阈值，baseMask为负，则表明是绿幕，为正则表明不是绿幕\r\n float baseMask = chromaDist - similarity;\r\n // 如果baseMask为负数，fullMask等于0；baseMask为正数，越大，则透明度越低\r\n float fullMask = pow(clamp(baseMask / smoothness, 0., 1.), 1.5);\r\n rgba.a = fullMask; // 设置透明度\r\n // 如果baseMask为负数，spillVal等于0；baseMask为整数，越小，饱和度越低\r\n float spillVal = pow(clamp(baseMask / spill, 0., 1.), 1.5);\r\n float desat = clamp(rgba.r * 0.2126 + rgba.g * 0.7152 + rgba.b * 0.0722, 0., 1.); // 计算当前像素的灰度值\r\n rgba.rgb = mix(vec3(desat, desat, desat), rgba.rgb, spillVal);\r\n FragColor = rgba;\r\n}\r\n`;\r\n\r\nconst POINT_POS = [-1, 1, -1, -1, 1, -1, 1, -1, 1, 1, -1, 1];\r\nconst TEX_COORD_POS = [0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1];\r\n\r\n// 初始化着色器程序，让 WebGL 知道如何绘制我们的数据\r\nfunction initShaderProgram(\r\n gl: WebGLRenderingContext,\r\n vsSource: string,\r\n fsSource: string,\r\n) {\r\n const vertexShader = loadShader(gl, gl.VERTEX_SHADER, vsSource)!;\r\n const fragmentShader = loadShader(gl, gl.FRAGMENT_SHADER, fsSource)!;\r\n\r\n // 创建着色器程序\r\n const shaderProgram = gl.createProgram()!;\r\n gl.attachShader(shaderProgram, vertexShader);\r\n gl.attachShader(shaderProgram, fragmentShader);\r\n gl.linkProgram(shaderProgram);\r\n\r\n if (!gl.getProgramParameter(shaderProgram, gl.LINK_STATUS)) {\r\n throw Error(\r\n gl.getProgramInfoLog(shaderProgram) ??\r\n 'Unable to initialize the shader program',\r\n );\r\n }\r\n\r\n return shaderProgram;\r\n}\r\n\r\n// 创建指定类型的着色器，上传 source 源码并编译\r\nfunction loadShader(gl: WebGLRenderingContext, type: number, source: string) {\r\n const shader = gl.createShader(type)!;\r\n\r\n // Send the source to the shader object\r\n gl.shaderSource(shader, source);\r\n\r\n // Compile the shader program\r\n gl.compileShader(shader);\r\n\r\n // See if it compiled successfully\r\n if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {\r\n const errMsg = gl.getShaderInfoLog(shader);\r\n gl.deleteShader(shader);\r\n throw Error(errMsg ?? 'An error occurred compiling the shaders');\r\n }\r\n\r\n return shader;\r\n}\r\n\r\nfunction updateTexture(\r\n gl: WebGLRenderingContext,\r\n img: TImgSource,\r\n texture: WebGLTexture,\r\n) {\r\n gl.bindTexture(gl.TEXTURE_2D, texture);\r\n gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, img);\r\n gl.drawArrays(gl.TRIANGLES, 0, 6);\r\n}\r\n\r\nfunction initTexture(gl: WebGLRenderingContext) {\r\n const texture = gl.createTexture();\r\n if (texture == null) throw Error('Create WebGL texture error');\r\n gl.bindTexture(gl.TEXTURE_2D, texture);\r\n\r\n // put a single pixel in the texture so we can use it immediately.\r\n const level = 0;\r\n const internalFormat = gl.RGBA;\r\n const width = 1;\r\n const height = 1;\r\n const border = 0;\r\n const srcFormat = gl.RGBA;\r\n const srcType = gl.UNSIGNED_BYTE;\r\n const pixel = new Uint8Array([0, 0, 255, 255]); // opaque blue\r\n gl.texImage2D(\r\n gl.TEXTURE_2D,\r\n level,\r\n internalFormat,\r\n width,\r\n height,\r\n border,\r\n srcFormat,\r\n srcType,\r\n pixel,\r\n );\r\n\r\n gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);\r\n gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);\r\n gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);\r\n gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);\r\n\r\n return texture;\r\n}\r\n\r\ninterface IChromakeyOpts {\r\n keyColor: [number, number, number];\r\n similarity: number;\r\n smoothness: number;\r\n spill: number;\r\n}\r\n\r\nfunction initCvs(\r\n opts: {\r\n width: number;\r\n height: number;\r\n } & IChromakeyOpts,\r\n) {\r\n const cvs =\r\n 'document' in globalThis\r\n ? globalThis.document.createElement('canvas')\r\n : new OffscreenCanvas(opts.width, opts.height);\r\n cvs.width = opts.width;\r\n cvs.height = opts.height;\r\n\r\n const gl = cvs.getContext('webgl2', {\r\n premultipliedAlpha: false,\r\n alpha: true,\r\n }) as WebGL2RenderingContext | null;\r\n\r\n if (gl == null) throw Error('Cant create gl context');\r\n\r\n const shaderProgram = initShaderProgram(gl, vertexShader, fragmentShader);\r\n gl.useProgram(shaderProgram);\r\n\r\n gl.uniform3fv(\r\n gl.getUniformLocation(shaderProgram, 'keyColor'),\r\n opts.keyColor.map((v) => v / 255),\r\n );\r\n gl.uniform1f(\r\n gl.getUniformLocation(shaderProgram, 'similarity'),\r\n opts.similarity,\r\n );\r\n gl.uniform1f(\r\n gl.getUniformLocation(shaderProgram, 'smoothness'),\r\n opts.smoothness,\r\n );\r\n gl.uniform1f(gl.getUniformLocation(shaderProgram, 'spill'), opts.spill);\r\n\r\n const posBuffer = gl.createBuffer();\r\n gl.bindBuffer(gl.ARRAY_BUFFER, posBuffer);\r\n gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(POINT_POS), gl.STATIC_DRAW);\r\n const a_position = gl.getAttribLocation(shaderProgram, 'a_position');\r\n gl.vertexAttribPointer(\r\n a_position,\r\n 2,\r\n gl.FLOAT,\r\n false,\r\n Float32Array.BYTES_PER_ELEMENT * 2,\r\n 0,\r\n );\r\n gl.enableVertexAttribArray(a_position);\r\n\r\n const texCoordBuffer = gl.createBuffer();\r\n gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);\r\n gl.bufferData(\r\n gl.ARRAY_BUFFER,\r\n new Float32Array(TEX_COORD_POS),\r\n gl.STATIC_DRAW,\r\n );\r\n const a_texCoord = gl.getAttribLocation(shaderProgram, 'a_texCoord');\r\n gl.vertexAttribPointer(\r\n a_texCoord,\r\n 2,\r\n gl.FLOAT,\r\n false,\r\n Float32Array.BYTES_PER_ELEMENT * 2,\r\n 0,\r\n );\r\n gl.enableVertexAttribArray(a_texCoord);\r\n\r\n gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, 1);\r\n\r\n return { cvs, gl };\r\n}\r\n\r\ntype TImgSource =\r\n | HTMLVideoElement\r\n | HTMLCanvasElement\r\n | HTMLImageElement\r\n | ImageBitmap\r\n | OffscreenCanvas\r\n | VideoFrame;\r\n\r\nfunction getSourceWH(imgSource: TImgSource) {\r\n return imgSource instanceof VideoFrame\r\n ? { width: imgSource.codedWidth, height: imgSource.codedHeight }\r\n : { width: imgSource.width, height: imgSource.height };\r\n}\r\n\r\nfunction getKeyColor(imgSource: TImgSource) {\r\n const cvs = new OffscreenCanvas(1, 1);\r\n const ctx = cvs.getContext('2d')!;\r\n ctx.drawImage(imgSource, 0, 0);\r\n const {\r\n data: [r, g, b],\r\n } = ctx.getImageData(0, 0, 1, 1);\r\n return [r, g, b] as [number, number, number];\r\n}\r\n\r\n/**\r\n * 绿幕抠图\r\n * keyColor 需要扣除的背景色，若不传则取第一个像素点\r\n * similarity 背景色相似度阈值，过小可能保留背景色，过大可能扣掉更多非背景像素点\r\n * smoothness 平滑度；过小可能出现锯齿，过大导致整体变透明\r\n * spill 饱和度；过小可能保留绿色混合，过大导致图片变灰度\r\n * @param opts: {\r\n * keyColor?: [r, g, b]\r\n * similarity: number\r\n * smoothness: number\r\n * spill: number\r\n * }\r\n */\r\nexport const createChromakey = (\r\n opts: Omit & {\r\n keyColor?: [number, number, number];\r\n },\r\n) => {\r\n let cvs: HTMLCanvasElement | OffscreenCanvas | null = null;\r\n let gl: WebGLRenderingContext | null = null;\r\n let keyC = opts.keyColor;\r\n let texture: WebGLTexture | null = null;\r\n\r\n return async (imgSource: TImgSource) => {\r\n if (cvs == null || gl == null || texture == null) {\r\n if (keyC == null) keyC = getKeyColor(imgSource);\r\n ({ cvs, gl } = initCvs({\r\n ...getSourceWH(imgSource),\r\n keyColor: keyC,\r\n ...opts,\r\n }));\r\n texture = initTexture(gl);\r\n }\r\n\r\n updateTexture(gl, imgSource, texture);\r\n\r\n if (\r\n globalThis.VideoFrame != null &&\r\n imgSource instanceof globalThis.VideoFrame\r\n ) {\r\n const rs = new VideoFrame(cvs, {\r\n alpha: 'keep',\r\n timestamp: imgSource.timestamp,\r\n duration: imgSource.duration ?? undefined,\r\n });\r\n imgSource.close();\r\n return rs;\r\n }\r\n\r\n return createImageBitmap(cvs, {\r\n imageOrientation: imgSource instanceof ImageBitmap ? 'flipY' : 'none',\r\n });\r\n };\r\n};\r\n","// 在主线程中执行的 工具函数\r\n\r\n/**\r\n * 创建一个新的 HTML 元素\r\n * @param tagName - 要创建的元素的标签名\r\n * @returns 新创建的 HTML 元素\r\n */\r\nexport function createEl(tagName: string): HTMLElement {\r\n return document.createElement(tagName);\r\n}\r\n\r\nfunction arrayBufferToBase64(buffer: ArrayBuffer) {\r\n var binary = '';\r\n var bytes = new Uint8Array(buffer);\r\n var len = bytes.byteLength;\r\n for (let i = 0; i < len; i++) {\r\n binary += String.fromCharCode(bytes[i]);\r\n }\r\n return window.btoa(binary);\r\n}\r\n\r\n/**\r\n * 将文本渲染为图片\r\n * @param txt - 要渲染的文本\r\n * @param cssText - 应用于文本的 CSS 样式\r\n * @returns 渲染后的图片元素\r\n */\r\nexport async function renderTxt2Img(\r\n txt: string,\r\n cssText: string,\r\n opts: {\r\n font?: { name: string; url: string };\r\n onCreated?: (el: HTMLElement) => void;\r\n } = {},\r\n): Promise {\r\n const div = createEl('pre');\r\n div.style.cssText = `margin: 0; ${cssText}; position: fixed;`;\r\n div.textContent = txt;\r\n document.body.appendChild(div);\r\n opts.onCreated?.(div);\r\n\r\n const { width, height } = div.getBoundingClientRect();\r\n // 计算出 rect，立即从dom移除\r\n div.remove();\r\n\r\n const img = new Image();\r\n img.width = width;\r\n img.height = height;\r\n const fontFaceStr =\r\n opts.font == null\r\n ? ''\r\n : `\r\n @font-face {\r\n font-family: '${opts.font.name}';\r\n src: url('data:font/woff2;base64,${arrayBufferToBase64(await (await fetch(opts.font.url)).arrayBuffer())}') format('woff2');\r\n }\r\n `;\r\n const svgStr = `\r\n \r\n \r\n \r\n
${div.outerHTML}
\r\n \r\n \r\n `\r\n .replace(/\\t/g, '')\r\n .replace(/#/g, '%23');\r\n\r\n img.src = `data:image/svg+xml;charset=utf-8,${svgStr}`;\r\n\r\n await new Promise((resolve) => {\r\n img.onload = resolve;\r\n });\r\n return img;\r\n}\r\n\r\n/**\r\n * 将文本渲染为 {@link ImageBitmap}，用来创建 {@link ImgClip}\r\n * @param txt - 要渲染的文本\r\n * @param cssText - 应用于文本的 CSS 样式\r\n * @param opts - 选项\r\n * @param opts.font - 自定义字体\r\n * @param opts.onCreated - 创建完成后的回调\r\n *\r\n * @example\r\n * new ImgClip(\r\n * await renderTxt2ImgBitmap(\r\n * '水印',\r\n * `font-size:40px; color: white; text-shadow: 2px 2px 6px red; font-family: CustomFont;`,\r\n * {\r\n * font: {\r\n * name: 'CustomFont',\r\n * url: '/CustomFont.ttf',\r\n * },\r\n * },\r\n * )\r\n * )\r\n */\r\nexport async function renderTxt2ImgBitmap(\r\n txt: string,\r\n cssText: string,\r\n opts: {\r\n font?: { name: string; url: string };\r\n onCreated?: (el: HTMLElement) => void;\r\n } = {},\r\n): Promise {\r\n const imgEl = await renderTxt2Img(txt, cssText, opts);\r\n const cvs = new OffscreenCanvas(imgEl.width, imgEl.height);\r\n const ctx = cvs.getContext('2d');\r\n ctx?.drawImage(imgEl, 0, 0, imgEl.width, imgEl.height);\r\n return await createImageBitmap(cvs);\r\n}\r\n","// 能同时在 worker 和主线程中运行的工具函数\n\nimport { workerTimer } from '@webrock/internal-utils';\nimport * as waveResampler from 'wave-resampler';\n\n/**\n * 合并（串联）多个 Float32Array，通常用于合并 PCM 数据\n */\nexport function concatFloat32Array(bufs: Float32Array[]): Float32Array {\n const rs = new Float32Array(\n bufs.map((buf) => buf.length).reduce((a, b) => a + b),\n );\n\n let offset = 0;\n for (const buf of bufs) {\n rs.set(buf, offset);\n offset += buf.length;\n }\n\n return rs;\n}\n\n/**\n * 将小片段的 PCM 合并成一个大片段\n * @param fragments 小片段 PCM，子元素是不同声道的原始 PCM 数据\n */\nexport function concatPCMFragments(\n fragments: Float32Array[][],\n): Float32Array[] {\n // fragments: [[chan0, chan1], [chan0, chan1]...]\n // chanListPCM: [[chan0, chan0...], [chan1, chan1...]]\n const chanListPCM: Float32Array[][] = [];\n for (let i = 0; i < fragments.length; i += 1) {\n for (let j = 0; j < fragments[i].length; j += 1) {\n if (chanListPCM[j] == null) chanListPCM[j] = [];\n chanListPCM[j].push(fragments[i][j]);\n }\n }\n // [bigChan0, bigChan1]\n return chanListPCM.map(concatFloat32Array);\n}\n\n/**\n * 从 AudioData 中提取 PCM 数据的工具函数\n */\nexport function extractPCM4AudioData(ad: AudioData): Float32Array[] {\n if (ad.format === 'f32-planar') {\n const rs = [];\n for (let idx = 0; idx < ad.numberOfChannels; idx += 1) {\n const chanBufSize = ad.allocationSize({ planeIndex: idx });\n const chanBuf = new ArrayBuffer(chanBufSize);\n ad.copyTo(chanBuf, { planeIndex: idx });\n rs.push(new Float32Array(chanBuf));\n }\n return rs;\n } else if (ad.format === 'f32') {\n const buf = new ArrayBuffer(ad.allocationSize({ planeIndex: 0 }));\n ad.copyTo(buf, { planeIndex: 0 });\n return convertF32ToPlanar(new Float32Array(buf), ad.numberOfChannels);\n } else if (ad.format === 's16') {\n const buf = new ArrayBuffer(ad.allocationSize({ planeIndex: 0 }));\n ad.copyTo(buf, { planeIndex: 0 });\n return convertS16ToF32Planar(new Int16Array(buf), ad.numberOfChannels);\n }\n throw Error('Unsupported audio data format');\n}\n\n/**\n * Convert s16 PCM to f32-planar\n * @param pcmS16Data - The s16 PCM data.\n * @param numChannels - Number of audio channels.\n * @returns An array of Float32Array, each containing the audio data for one channel.\n */\nfunction convertS16ToF32Planar(pcmS16Data: Int16Array, numChannels: number) {\n const numSamples = pcmS16Data.length / numChannels;\n const planarData = Array.from(\n { length: numChannels },\n () => new Float32Array(numSamples),\n );\n\n for (let i = 0; i < numSamples; i++) {\n for (let channel = 0; channel < numChannels; channel++) {\n const sample = pcmS16Data[i * numChannels + channel];\n planarData[channel][i] = sample / 32768; // Normalize to range [-1.0, 1.0]\n }\n }\n\n return planarData;\n}\n\nfunction convertF32ToPlanar(pcmF32Data: Float32Array, numChannels: number) {\n const numSamples = pcmF32Data.length / numChannels;\n const planarData = Array.from(\n { length: numChannels },\n () => new Float32Array(numSamples),\n );\n\n for (let i = 0; i < numSamples; i++) {\n for (let channel = 0; channel < numChannels; channel++) {\n planarData[channel][i] = pcmF32Data[i * numChannels + channel];\n }\n }\n\n return planarData;\n}\n\n/**\n * 从 AudioBuffer 中提取 PCM\n */\nexport function extractPCM4AudioBuffer(ab: AudioBuffer): Float32Array[] {\n return Array(ab.numberOfChannels)\n .fill(0)\n .map((_, idx) => {\n return ab.getChannelData(idx);\n });\n}\n\n/**\n * 调整音频数据的音量\n * @param ad - 要调整的音频对象\n * @param volume - 音量调整系数（0.0 - 1.0）\n * @returns 调整音量后的新音频数据\n */\nexport function adjustAudioDataVolume(ad: AudioData, volume: number) {\n const data = new Float32Array(\n concatFloat32Array(extractPCM4AudioData(ad)),\n ).map((v) => v * volume);\n const newAd = new AudioData({\n sampleRate: ad.sampleRate,\n numberOfChannels: ad.numberOfChannels,\n timestamp: ad.timestamp,\n format: ad.format!,\n numberOfFrames: ad.numberOfFrames,\n data,\n });\n ad.close();\n return newAd;\n}\n\n/**\n * 解码图像流，返回一个视频帧数组。\n *\n * @param stream - 包含图像数据的可读流。\n * @param type - 图像的 MIME 类型，例如 'image/jpeg'。\n *\n * @returns 返回一个 Promise，该 Promise 在解码完成后解析为 {@link VideoFrame} 数组。\n *\n * @see [解码动图](https://webav-tech.github.io/WebAV/demo/1_3-decode-image)\n *\n * @example\n *\n * const frames = await decodeImg(\n * (await fetch('')).body!,\n * `image/gif`,\n * );\n */\nexport async function decodeImg(\n stream: ReadableStream,\n type: string,\n): Promise {\n const init = {\n type,\n data: stream,\n };\n const imageDecoder = new ImageDecoder(init);\n\n await Promise.all([imageDecoder.completed, imageDecoder.tracks.ready]);\n\n let frameCnt = imageDecoder.tracks.selectedTrack?.frameCount ?? 1;\n\n const rs: VideoFrame[] = [];\n for (let i = 0; i < frameCnt; i += 1) {\n rs.push((await imageDecoder.decode({ frameIndex: i })).image);\n }\n return rs;\n}\n\n/**\n * 混合双通道音轨的 PCM 数据，并将多声道并排成一个 Float32Array 输出\n * @param audios - 一个二维数组，每个元素是一个 Float32Array 数组，代表一个音频流的 PCM 数据。\n * 每个 Float32Array 数组的第一个元素是左声道数据，第二个元素（如果有）是右声道数据。\n * 如果只有左声道数据，则右声道将复用左声道数据。\n *\n * @returns 返回一个 Float32Array，返回结果是将这个一个音轨的左右声道并排成 Float32Array。\n *\n * @example\n *\n * const audios = [\n * [new Float32Array([0.1, 0.2, 0.3]), new Float32Array([0.4, 0.5, 0.6])],\n * [new Float32Array([0.7, 0.8, 0.9])],\n * ];\n * const mixed = mixinPCM(audios);\n */\nexport function mixinPCM(audios: Float32Array[][]): Float32Array {\n const maxLen = Math.max(...audios.map((a) => a[0]?.length ?? 0));\n const data = new Float32Array(maxLen * 2);\n\n for (let bufIdx = 0; bufIdx < maxLen; bufIdx++) {\n let chan0 = 0;\n let chan1 = 0;\n for (let trackIdx = 0; trackIdx < audios.length; trackIdx++) {\n const _c0 = audios[trackIdx][0]?.[bufIdx] ?? 0;\n // 如果是单声道 PCM，第二声道复用第一声道数据\n const _c1 = audios[trackIdx][1]?.[bufIdx] ?? _c0;\n chan0 += _c0;\n chan1 += _c1;\n }\n data[bufIdx] = chan0;\n data[bufIdx + maxLen] = chan1;\n }\n\n return data;\n}\n\n/**\n * 对 PCM 音频数据进行重采样。\n *\n * @param pcmData - 一个 Float32Array 数组，每个元素代表一个声道的 PCM 数据。\n * @param curRate - 当前的采样率。\n * @param target - 目标参数对象。\n * @param target.rate - 目标采样率。\n * @param target.chanCount - 目标声道数。\n *\n * @returns 返回一个 Promise，该 Promise 在重采样完成后解析为一个 Float32Array 数组，每个元素代表一个声道的 PCM 数据。\n *\n * @example\n *\n * const pcmData = [new Float32Array([0.1, 0.2, 0.3]), new Float32Array([0.4, 0.5, 0.6])];\n * const curRate = 44100;\n * const target = { rate: 48000, chanCount: 2 };\n * const resampled = await audioResample(pcmData, curRate, target);\n */\nexport async function audioResample(\n pcmData: Float32Array[],\n curRate: number,\n target: {\n rate: number;\n chanCount: number;\n },\n): Promise {\n const chanCnt = pcmData.length;\n const emptyPCM = Array(target.chanCount)\n .fill(0)\n .map(() => new Float32Array(0));\n if (chanCnt === 0) return emptyPCM;\n\n const len = Math.max(...pcmData.map((c) => c.length));\n if (len === 0) return emptyPCM;\n\n // The Worker scope does not have access to OfflineAudioContext\n if (globalThis.OfflineAudioContext == null) {\n return pcmData.map(\n (p) =>\n new Float32Array(\n waveResampler.resample(p, curRate, target.rate, {\n method: 'sinc',\n LPF: false,\n }),\n ),\n );\n }\n\n const ctx = new globalThis.OfflineAudioContext(\n target.chanCount,\n (len * target.rate) / curRate,\n target.rate,\n );\n const abSource = ctx.createBufferSource();\n const ab = ctx.createBuffer(chanCnt, len, curRate);\n pcmData.forEach((d, idx) => ab.copyToChannel(d, idx));\n\n abSource.buffer = ab;\n abSource.connect(ctx.destination);\n abSource.start();\n\n return extractPCM4AudioBuffer(await ctx.startRendering());\n}\n\n/**\n * 使当前执行环境暂停一段时间。\n * @param time - 暂停的时间，单位为毫秒。\n * @example\n * await sleep(1000); // 暂停 1 秒\n */\nexport function sleep(time: number): Promise {\n return new Promise((resolve) => {\n const stop = workerTimer(() => {\n stop();\n resolve();\n }, time);\n });\n}\n\n/**\n * 从给定的 Float32Array 中提取一个环形切片，超出边界从 0 开始循环\n *\n * 主要用于截取 PCM 实现循环播放\n *\n * @param data - 输入的 Float32Array。\n * @param start - 切片的开始索引。\n * @param end - 切片的结束索引。\n * @returns - 返回一个新的 Float32Array，包含从 start 到 end 的数据。\n *\n * @example\n * const data = new Float32Array([0, 1, 2, 3, 4, 5]);\n * ringSliceFloat32Array(data, 4, 6); // => Float32Array [4, 5, 0]\n */\nexport function ringSliceFloat32Array(\n data: Float32Array,\n start: number,\n end: number,\n): Float32Array {\n const cnt = end - start;\n const rs = new Float32Array(cnt);\n let i = 0;\n while (i < cnt) {\n rs[i] = data[(start + i) % data.length];\n i += 1;\n }\n return rs;\n}\n\n/**\n * 改变 PCM 数据的播放速率，1 表示正常播放，0.5 表示播放速率减半，2 表示播放速率加倍\n */\nexport function changePCMPlaybackRate(\n pcmData: Float32Array,\n playbackRate: number,\n) {\n // 计算新的采样率\n const newLength = Math.floor(pcmData.length / playbackRate);\n const newPcmData = new Float32Array(newLength);\n\n // 线性插值\n for (let i = 0; i < newLength; i++) {\n // 原始数据中的位置\n const originalIndex = i * playbackRate;\n const intIndex = Math.floor(originalIndex);\n const frac = originalIndex - intIndex;\n\n // 边界检查\n if (intIndex + 1 < pcmData.length) {\n newPcmData[i] =\n pcmData[intIndex] * (1 - frac) + pcmData[intIndex + 1] * frac;\n } else {\n newPcmData[i] = pcmData[intIndex]; // 最后一个样本\n }\n }\n\n return newPcmData;\n}\n","interface IClipMeta {\r\n width: number;\r\n height: number;\r\n duration: number;\r\n}\r\n\r\n/**\r\n * 所有素材需要实现的接口\r\n *\r\n * 素材（Clip）是不同数据类型的抽象，给其他模块提供数据\r\n *\r\n * WebAV 内置了 {@link MP4Clip}, {@link AudioClip}, {@link ImgClip}, {@link MediaStreamClip} 等常用素材，用于给 {@link Combinator} {@link AVCanvas} 提供数据\r\n *\r\n * 你只需实现该接口即可自定义素材，拥有最大的灵活度来生成视频内容，比如动画、转场效果等\r\n * @see [自定义素材](https://webav-tech.github.io/WebAV/demo/2_6-custom-clip)\r\n *\r\n */\r\nexport interface IClip {\r\n /**\r\n * 从素材中提取指定时间数据\r\n * @param time 时间，单位 微秒\r\n */\r\n tick: (time: number) => Promise<{\r\n video?: VideoFrame | ImageBitmap | null;\r\n audio?: Float32Array[];\r\n state: 'done' | 'success';\r\n }>;\r\n\r\n /**\r\n * 当素材准备完成，ready 会切换到 resolved 状态\r\n */\r\n readonly ready: Promise;\r\n\r\n /**\r\n * 数据元数据\r\n */\r\n readonly meta: IClipMeta;\r\n\r\n /**\r\n * clone，返回一个新素材\r\n */\r\n clone: () => Promise;\r\n\r\n /**\r\n * 按指定时间切割，返回该时刻前后两个新素材，常用于剪辑场景按时间分割素材\r\n *\r\n * 该方法不会破坏原素材的数据\r\n *\r\n * @param time 时间，微秒\r\n * @returns\r\n */\r\n split?: (time: number) => Promise<[this, this]>;\r\n\r\n /**\r\n * 销毁实例，释放资源\r\n */\r\n destroy: () => void;\r\n}\r\n\r\n/**\r\n * 默认的音频设置，⚠️ 不要变更它的值 ⚠️\r\n */\r\nexport const DEFAULT_AUDIO_CONF = {\r\n sampleRate: 48000,\r\n channelCount: 2,\r\n codec: 'mp4a.40.2',\r\n} as const;\r\n","import mp4box, {\n AudioTrackOpts,\n ESDSBoxParser,\n MP4ABoxParser,\n MP4ArrayBuffer,\n MP4File,\n MP4Info,\n MP4Sample,\n TrakBoxParser,\n VideoTrackOpts,\n} from '@webav/mp4box.js';\nimport { file } from 'opfs-tools';\nimport { DEFAULT_AUDIO_CONF } from '../clips';\n\nexport function extractFileConfig(file: MP4File, info: MP4Info) {\n const vTrack = info.videoTracks[0];\n const rs: {\n videoTrackConf?: VideoTrackOpts;\n videoDecoderConf?: Parameters[0];\n audioTrackConf?: AudioTrackOpts;\n audioDecoderConf?: Parameters[0];\n } = {};\n if (vTrack != null) {\n const videoDesc = parseVideoCodecDesc(file.getTrackById(vTrack.id)).buffer;\n const { descKey, type } = vTrack.codec.startsWith('avc1')\n ? { descKey: 'avcDecoderConfigRecord', type: 'avc1' }\n : vTrack.codec.startsWith('hvc1')\n ? { descKey: 'hevcDecoderConfigRecord', type: 'hvc1' }\n : { descKey: '', type: '' };\n if (descKey !== '') {\n rs.videoTrackConf = {\n timescale: vTrack.timescale,\n duration: vTrack.duration,\n width: vTrack.video.width,\n height: vTrack.video.height,\n brands: info.brands,\n type,\n [descKey]: videoDesc,\n };\n }\n\n rs.videoDecoderConf = {\n codec: vTrack.codec,\n codedHeight: vTrack.video.height,\n codedWidth: vTrack.video.width,\n description:\n videoDesc instanceof Uint8Array ? videoDesc : new Uint8Array(videoDesc),\n };\n }\n\n const aTrack = info.audioTracks[0];\n if (aTrack != null) {\n const esdsBox = getESDSBoxFromMP4File(file);\n rs.audioTrackConf = {\n timescale: aTrack.timescale,\n samplerate: aTrack.audio.sample_rate,\n channel_count: aTrack.audio.channel_count,\n hdlr: 'soun',\n type: aTrack.codec.startsWith('mp4a') ? 'mp4a' : aTrack.codec,\n description: getESDSBoxFromMP4File(file),\n };\n rs.audioDecoderConf = {\n codec: aTrack.codec.startsWith('mp4a')\n ? DEFAULT_AUDIO_CONF.codec\n : aTrack.codec,\n numberOfChannels: aTrack.audio.channel_count,\n sampleRate: aTrack.audio.sample_rate,\n ...(esdsBox == null ? {} : parseAudioInfo4ESDSBox(esdsBox)),\n };\n }\n return rs;\n}\n\n// track is H.264, H.265 or VPX.\nfunction parseVideoCodecDesc(track: TrakBoxParser): Uint8Array {\n for (const entry of track.mdia.minf.stbl.stsd.entries) {\n // @ts-expect-error\n const box = entry.avcC ?? entry.hvcC ?? entry.av1C ?? entry.vpcC;\n if (box != null) {\n const stream = new mp4box.DataStream(\n undefined,\n 0,\n mp4box.DataStream.BIG_ENDIAN,\n );\n box.write(stream);\n return new Uint8Array(stream.buffer.slice(8)); // Remove the box header.\n }\n }\n throw Error('avcC, hvcC, av1C or VPX not found');\n}\n\nfunction getESDSBoxFromMP4File(file: MP4File, codec = 'mp4a') {\n const mp4aBox = file.moov?.traks\n .map((t) => t.mdia.minf.stbl.stsd.entries)\n .flat()\n .find(({ type }) => type === codec) as MP4ABoxParser;\n\n return mp4aBox?.esds;\n}\n\n// 解决封装层音频信息标识错误，导致解码异常\nfunction parseAudioInfo4ESDSBox(esds: ESDSBoxParser) {\n const decoderConf = esds.esd.descs[0]?.descs[0];\n if (decoderConf == null) return {};\n\n const [byte1, byte2] = decoderConf.data;\n // sampleRate 是第一字节后 3bit + 第二字节前 1bit\n const sampleRateIdx = ((byte1 & 0x07) << 1) + (byte2 >> 7);\n // numberOfChannels 是第二字节 [2, 5] 4bit\n const numberOfChannels = (byte2 & 0x7f) >> 3;\n const sampleRateEnum = [\n 96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025,\n 8000, 7350,\n ] as const;\n return {\n sampleRate: sampleRateEnum[sampleRateIdx],\n numberOfChannels,\n };\n}\n\n/**\n * 快速解析 mp4 文件，如果是非 fMP4 格式，会优先解析 moov box（略过 mdat）避免占用过多内存\n */\nexport async function quickParseMP4File(\n reader: Awaited['createReader']>>,\n onReady: (data: { mp4boxFile: MP4File; info: MP4Info }) => void,\n onSamples: (\n id: number,\n sampleType: 'video' | 'audio',\n samples: MP4Sample[],\n ) => void,\n) {\n const mp4boxFile = mp4box.createFile(false);\n mp4boxFile.onReady = (info) => {\n onReady({ mp4boxFile, info });\n const vTrackId = info.videoTracks[0]?.id;\n if (vTrackId != null)\n mp4boxFile.setExtractionOptions(vTrackId, 'video', { nbSamples: 100 });\n\n const aTrackId = info.audioTracks[0]?.id;\n if (aTrackId != null)\n mp4boxFile.setExtractionOptions(aTrackId, 'audio', { nbSamples: 100 });\n\n mp4boxFile.start();\n };\n mp4boxFile.onSamples = onSamples;\n\n await parse();\n\n async function parse() {\n let cursor = 0;\n const maxReadSize = 30 * 1024 * 1024;\n while (true) {\n const data = (await reader.read(maxReadSize, {\n at: cursor,\n })) as MP4ArrayBuffer;\n if (data.byteLength === 0) break;\n data.fileStart = cursor;\n const nextPos = mp4boxFile.appendBuffer(data);\n if (nextPos == null) break;\n cursor = nextPos;\n }\n\n mp4boxFile.stop();\n }\n}\n","import { MP4Info, MP4Sample } from '@webav/mp4box.js';\nimport { Log } from '@webrock/internal-utils';\nimport { file, tmpfile, write } from 'opfs-tools';\nimport { audioResample, extractPCM4AudioData, sleep } from '../av-utils';\nimport {\n extractFileConfig,\n quickParseMP4File,\n} from '../mp4-utils/mp4box-utils';\nimport { DEFAULT_AUDIO_CONF, IClip } from './iclip';\n\nlet CLIP_ID = 0;\n\ntype OPFSToolFile = ReturnType;\nfunction isOTFile(obj: any): obj is OPFSToolFile {\n return obj.kind === 'file' && obj.createReader instanceof Function;\n}\n\n// 用于内部创建 MP4Clip 实例\ntype MPClipCloneArgs = Awaited> & {\n localFile: OPFSToolFile;\n};\n\ninterface MP4DecoderConf {\n video: VideoDecoderConfig | null;\n audio: AudioDecoderConfig | null;\n}\n\ninterface MP4ClipOpts {\n audio?: boolean | { volume: number };\n /**\n * 不安全，随时可能废弃\n */\n __unsafe_hardwareAcceleration__?: HardwarePreference;\n}\n\ntype ExtMP4Sample = Omit & {\n is_idr: boolean;\n deleted?: boolean;\n data: null | Uint8Array;\n};\n\ntype LocalFileReader = Awaited>;\n\ntype ThumbnailOpts = {\n start: number;\n end: number;\n step: number;\n};\n\n/**\n * MP4 素材，解析 MP4 文件，使用 {@link MP4Clip.tick} 按需解码指定时间的图像帧\n *\n * 可用于实现视频抽帧、生成缩略图、视频编辑等功能\n *\n * @example\n * new MP4Clip((await fetch('')).body)\n * new MP4Clip(mp4File.stream())\n *\n * @see {@link Combinator}\n * @see [AVCanvas](../../av-canvas/classes/AVCanvas.html)\n *\n * @see [解码播放视频](https://webav-tech.github.io/WebAV/demo/1_1-decode-video)\n */\nexport class MP4Clip implements IClip {\n #insId = CLIP_ID++;\n\n #log = Log.create(`MP4Clip id:${this.#insId},`);\n\n ready: IClip['ready'];\n\n #destroyed = false;\n\n #meta = {\n // 微秒\n duration: 0,\n width: 0,\n height: 0,\n audioSampleRate: 0,\n audioChanCount: 0,\n };\n\n get meta() {\n return { ...this.#meta };\n }\n\n #localFile: OPFSToolFile;\n\n #headerBoxPos: Array<{ start: number; size: number }> = [];\n /**\n * 提供视频头（box: ftyp, moov）的二进制数据\n * 使用任意 mp4 demxer 解析即可获得详细的视频信息\n * 单元测试包含使用 mp4box.js 解析示例代码\n */\n async getFileHeaderBinData() {\n await this.ready;\n const oFile = await this.#localFile.getOriginFile();\n if (oFile == null) throw Error('MP4Clip localFile is not origin file');\n\n return await new Blob(\n this.#headerBoxPos.map(({ start, size }) =>\n oFile.slice(start, start + size),\n ),\n ).arrayBuffer();\n }\n\n #volume = 1;\n\n #videoSamples: ExtMP4Sample[] = [];\n\n #audioSamples: ExtMP4Sample[] = [];\n\n #videoFrameFinder: VideoFrameFinder | null = null;\n #audioFrameFinder: AudioFrameFinder | null = null;\n\n #decoderConf: {\n video: VideoDecoderConfig | null;\n audio: AudioDecoderConfig | null;\n } = {\n video: null,\n audio: null,\n };\n\n #opts: MP4ClipOpts = { audio: true };\n\n constructor(\n source: OPFSToolFile | ReadableStream | MPClipCloneArgs,\n opts: MP4ClipOpts = {},\n ) {\n if (\n !(source instanceof ReadableStream) &&\n !isOTFile(source) &&\n !Array.isArray(source.videoSamples)\n ) {\n throw Error('Illegal argument');\n }\n\n this.#opts = { audio: true, ...opts };\n this.#volume =\n typeof opts.audio === 'object' && 'volume' in opts.audio\n ? opts.audio.volume\n : 1;\n\n const initByStream = async (s: ReadableStream) => {\n await write(this.#localFile, s);\n return this.#localFile;\n };\n\n this.#localFile = isOTFile(source)\n ? source\n : 'localFile' in source\n ? source.localFile // from clone\n : tmpfile();\n\n this.ready = (\n source instanceof ReadableStream\n ? initByStream(source).then((otFile) =>\n mp4FileToSamples(otFile, this.#opts),\n )\n : isOTFile(source)\n ? mp4FileToSamples(source, this.#opts)\n : Promise.resolve(source)\n ).then(\n async ({ videoSamples, audioSamples, decoderConf, headerBoxPos }) => {\n this.#videoSamples = videoSamples;\n this.#audioSamples = audioSamples;\n this.#decoderConf = decoderConf;\n this.#headerBoxPos = headerBoxPos;\n\n const { videoFrameFinder, audioFrameFinder } = genDecoder(\n {\n video:\n decoderConf.video == null\n ? null\n : {\n ...decoderConf.video,\n hardwareAcceleration:\n this.#opts.__unsafe_hardwareAcceleration__,\n },\n audio: decoderConf.audio,\n },\n await this.#localFile.createReader(),\n videoSamples,\n audioSamples,\n this.#opts.audio !== false ? this.#volume : 0,\n );\n this.#videoFrameFinder = videoFrameFinder;\n this.#audioFrameFinder = audioFrameFinder;\n\n this.#meta = genMeta(decoderConf, videoSamples, audioSamples);\n this.#log.info('MP4Clip meta:', this.#meta);\n return { ...this.#meta };\n },\n );\n }\n\n /**\n * 拦截 {@link MP4Clip.tick} 方法返回的数据，用于对图像、音频数据二次处理\n * @param time 调用 tick 的时间\n * @param tickRet tick 返回的数据\n *\n * @see [移除视频绿幕背景](https://webav-tech.github.io/WebAV/demo/3_2-chromakey-video)\n */\n tickInterceptor: >>(\n time: number,\n tickRet: T,\n ) => Promise = async (_, tickRet) => tickRet;\n\n /**\n * 获取素材指定时刻的图像帧、音频数据\n * @param time 微秒\n */\n async tick(time: number): Promise<{\n video?: VideoFrame;\n audio: Float32Array[];\n state: 'success' | 'done';\n }> {\n if (time >= this.#meta.duration) {\n return await this.tickInterceptor(time, {\n audio: (await this.#audioFrameFinder?.find(time)) ?? [],\n state: 'done',\n });\n }\n\n const [audio, video] = await Promise.all([\n this.#audioFrameFinder?.find(time) ?? [],\n this.#videoFrameFinder?.find(time),\n ]);\n\n if (video == null) {\n return await this.tickInterceptor(time, {\n audio,\n state: 'success',\n });\n }\n\n return await this.tickInterceptor(time, {\n video,\n audio,\n state: 'success',\n });\n }\n\n #thumbAborter = new AbortController();\n /**\n * 生成缩略图，默认每个关键帧生成一个 100px 宽度的缩略图。\n *\n * @param imgWidth 缩略图宽度，默认 100\n * @param opts Partial\n * @returns Promise>\n */\n async thumbnails(\n imgWidth = 100,\n opts?: Partial,\n ): Promise> {\n this.#thumbAborter.abort();\n this.#thumbAborter = new AbortController();\n const aborterSignal = this.#thumbAborter.signal;\n\n await this.ready;\n const abortMsg = 'generate thumbnails aborted';\n if (aborterSignal.aborted) throw Error(abortMsg);\n\n const { width, height } = this.#meta;\n const convtr = createVF2BlobConvtr(\n imgWidth,\n Math.round(height * (imgWidth / width)),\n { quality: 0.1, type: 'image/png' },\n );\n\n return new Promise>(\n async (resolve, reject) => {\n let pngPromises: Array<{ ts: number; img: Promise }> = [];\n const vc = this.#decoderConf.video;\n if (vc == null || this.#videoSamples.length === 0) {\n resolver();\n return;\n }\n aborterSignal.addEventListener('abort', () => {\n reject(Error(abortMsg));\n });\n\n async function resolver() {\n if (aborterSignal.aborted) return;\n resolve(\n await Promise.all(\n pngPromises.map(async (it) => ({\n ts: it.ts,\n img: await it.img,\n })),\n ),\n );\n }\n\n function pushPngPromise(vf: VideoFrame) {\n pngPromises.push({\n ts: vf.timestamp,\n img: convtr(vf),\n });\n }\n\n const { start = 0, end = this.#meta.duration, step } = opts ?? {};\n if (step) {\n let cur = start;\n // 创建一个新的 VideoFrameFinder 实例，避免与 tick 方法共用而导致冲突\n const videoFrameFinder = new VideoFrameFinder(\n await this.#localFile.createReader(),\n this.#videoSamples,\n {\n ...vc,\n hardwareAcceleration: this.#opts.__unsafe_hardwareAcceleration__,\n },\n );\n while (cur <= end && !aborterSignal.aborted) {\n const vf = await videoFrameFinder.find(cur);\n if (vf) pushPngPromise(vf);\n cur += step;\n }\n videoFrameFinder.destroy();\n resolver();\n } else {\n await thumbnailByKeyFrame(\n this.#videoSamples,\n this.#localFile,\n vc,\n aborterSignal,\n { start, end },\n (vf, done) => {\n if (vf != null) pushPngPromise(vf);\n if (done) resolver();\n },\n );\n }\n },\n );\n }\n\n async split(time: number) {\n await this.ready;\n\n if (time <= 0 || time >= this.#meta.duration)\n throw Error('\"time\" out of bounds');\n\n const [preVideoSlice, postVideoSlice] = splitVideoSampleByTime(\n this.#videoSamples,\n time,\n );\n const [preAudioSlice, postAudioSlice] = splitAudioSampleByTime(\n this.#audioSamples,\n time,\n );\n const preClip = new MP4Clip(\n {\n localFile: this.#localFile,\n videoSamples: preVideoSlice ?? [],\n audioSamples: preAudioSlice ?? [],\n decoderConf: this.#decoderConf,\n headerBoxPos: this.#headerBoxPos,\n },\n this.#opts,\n );\n const postClip = new MP4Clip(\n {\n localFile: this.#localFile,\n videoSamples: postVideoSlice ?? [],\n audioSamples: postAudioSlice ?? [],\n decoderConf: this.#decoderConf,\n headerBoxPos: this.#headerBoxPos,\n },\n this.#opts,\n );\n await Promise.all([preClip.ready, postClip.ready]);\n\n return [preClip, postClip] as [this, this];\n }\n\n async clone() {\n await this.ready;\n const clip = new MP4Clip(\n {\n localFile: this.#localFile,\n videoSamples: [...this.#videoSamples],\n audioSamples: [...this.#audioSamples],\n decoderConf: this.#decoderConf,\n headerBoxPos: this.#headerBoxPos,\n },\n this.#opts,\n );\n await clip.ready;\n clip.tickInterceptor = this.tickInterceptor;\n return clip as this;\n }\n\n /**\n * 拆分 MP4Clip 为仅包含视频轨道和音频轨道的 MP4Clip\n * @returns Mp4CLip[]\n */\n async splitTrack() {\n await this.ready;\n const clips: MP4Clip[] = [];\n if (this.#videoSamples.length > 0) {\n const videoClip = new MP4Clip(\n {\n localFile: this.#localFile,\n videoSamples: [...this.#videoSamples],\n audioSamples: [],\n decoderConf: {\n video: this.#decoderConf.video,\n audio: null,\n },\n headerBoxPos: this.#headerBoxPos,\n },\n this.#opts,\n );\n await videoClip.ready;\n videoClip.tickInterceptor = this.tickInterceptor;\n clips.push(videoClip);\n }\n if (this.#audioSamples.length > 0) {\n const audioClip = new MP4Clip(\n {\n localFile: this.#localFile,\n videoSamples: [],\n audioSamples: [...this.#audioSamples],\n decoderConf: {\n audio: this.#decoderConf.audio,\n video: null,\n },\n headerBoxPos: this.#headerBoxPos,\n },\n this.#opts,\n );\n await audioClip.ready;\n audioClip.tickInterceptor = this.tickInterceptor;\n clips.push(audioClip);\n }\n\n return clips;\n }\n\n destroy(): void {\n if (this.#destroyed) return;\n this.#log.info('MP4Clip destroy');\n this.#destroyed = true;\n\n this.#videoFrameFinder?.destroy();\n this.#audioFrameFinder?.destroy();\n }\n}\n\nfunction genMeta(\n decoderConf: MP4DecoderConf,\n videoSamples: ExtMP4Sample[],\n audioSamples: ExtMP4Sample[],\n) {\n const meta = {\n duration: 0,\n width: 0,\n height: 0,\n audioSampleRate: 0,\n audioChanCount: 0,\n };\n if (decoderConf.video != null && videoSamples.length > 0) {\n meta.width = decoderConf.video.codedWidth ?? 0;\n meta.height = decoderConf.video.codedHeight ?? 0;\n }\n if (decoderConf.audio != null && audioSamples.length > 0) {\n meta.audioSampleRate = DEFAULT_AUDIO_CONF.sampleRate;\n meta.audioChanCount = DEFAULT_AUDIO_CONF.channelCount;\n }\n\n let vDuration = 0;\n let aDuration = 0;\n if (videoSamples.length > 0) {\n for (let i = videoSamples.length - 1; i >= 0; i--) {\n const s = videoSamples[i];\n if (s.deleted) continue;\n vDuration = s.cts + s.duration;\n break;\n }\n }\n if (audioSamples.length > 0) {\n const lastSampele = audioSamples.at(-1)!;\n aDuration = lastSampele.cts + lastSampele.duration;\n }\n meta.duration = Math.max(vDuration, aDuration);\n\n return meta;\n}\n\nfunction genDecoder(\n decoderConf: MP4DecoderConf,\n localFileReader: LocalFileReader,\n videoSamples: ExtMP4Sample[],\n audioSamples: ExtMP4Sample[],\n volume: number,\n) {\n return {\n audioFrameFinder:\n volume === 0 || decoderConf.audio == null || audioSamples.length === 0\n ? null\n : new AudioFrameFinder(\n localFileReader,\n audioSamples,\n decoderConf.audio,\n {\n volume,\n targetSampleRate: DEFAULT_AUDIO_CONF.sampleRate,\n },\n ),\n videoFrameFinder:\n decoderConf.video == null || videoSamples.length === 0\n ? null\n : new VideoFrameFinder(\n localFileReader,\n videoSamples,\n decoderConf.video,\n ),\n };\n}\n\nasync function mp4FileToSamples(otFile: OPFSToolFile, opts: MP4ClipOpts = {}) {\n let mp4Info: MP4Info | null = null;\n const decoderConf: MP4DecoderConf = { video: null, audio: null };\n let videoSamples: ExtMP4Sample[] = [];\n let audioSamples: ExtMP4Sample[] = [];\n let headerBoxPos: Array<{ start: number; size: number }> = [];\n\n let videoDeltaTS = -1;\n let audioDeltaTS = -1;\n const reader = await otFile.createReader();\n await quickParseMP4File(\n reader,\n (data) => {\n mp4Info = data.info;\n const ftyp = data.mp4boxFile.ftyp!;\n headerBoxPos.push({ start: ftyp.start, size: ftyp.size });\n const moov = data.mp4boxFile.moov!;\n headerBoxPos.push({ start: moov.start, size: moov.size });\n\n let { videoDecoderConf: vc, audioDecoderConf: ac } = extractFileConfig(\n data.mp4boxFile,\n data.info,\n );\n decoderConf.video = vc ?? null;\n decoderConf.audio = ac ?? null;\n if (vc == null && ac == null) {\n Log.error('MP4Clip no video and audio track');\n }\n Log.info(\n 'mp4BoxFile moov ready',\n {\n ...data.info,\n tracks: null,\n videoTracks: null,\n audioTracks: null,\n },\n decoderConf,\n );\n },\n (_, type, samples) => {\n if (type === 'video') {\n if (videoDeltaTS === -1) videoDeltaTS = samples[0].dts;\n for (const s of samples) {\n videoSamples.push(normalizeTimescale(s, videoDeltaTS, 'video'));\n }\n } else if (type === 'audio' && opts.audio) {\n if (audioDeltaTS === -1) audioDeltaTS = samples[0].dts;\n for (const s of samples) {\n audioSamples.push(normalizeTimescale(s, audioDeltaTS, 'audio'));\n }\n }\n },\n );\n await reader.close();\n\n const lastSampele = videoSamples.at(-1) ?? audioSamples.at(-1);\n if (mp4Info == null) {\n throw Error('MP4Clip stream is done, but not emit ready');\n } else if (lastSampele == null) {\n throw Error('MP4Clip stream not contain any sample');\n }\n // 修复首帧黑帧\n fixFirstBlackFrame(videoSamples);\n Log.info('mp4 stream parsed');\n return {\n videoSamples,\n audioSamples,\n decoderConf,\n headerBoxPos,\n };\n\n function normalizeTimescale(\n s: MP4Sample,\n delta = 0,\n sampleType: 'video' | 'audio',\n ) {\n // todo: perf 丢弃多余字段，小尺寸对象性能更好\n const idrOffset =\n sampleType === 'video' && s.is_sync\n ? idrNALUOffset(s.data, s.description.type)\n : -1;\n let offset = s.offset;\n let size = s.size;\n if (idrOffset >= 0) {\n // 当 IDR 帧前面携带 SEI 数据可能导致解码失败\n // 所以此处通过控制 offset、size 字段 跳过 SEI 数据\n offset += idrOffset;\n size -= idrOffset;\n }\n return {\n ...s,\n is_idr: idrOffset >= 0,\n offset,\n size,\n cts: ((s.cts - delta) / s.timescale) * 1e6,\n dts: ((s.dts - delta) / s.timescale) * 1e6,\n duration: (s.duration / s.timescale) * 1e6,\n timescale: 1e6,\n // 音频数据量可控，直接保存在内存中\n data: sampleType === 'video' ? null : s.data,\n };\n }\n}\n\nclass VideoFrameFinder {\n #dec: VideoDecoder | null = null;\n constructor(\n public localFileReader: LocalFileReader,\n public samples: ExtMP4Sample[],\n public conf: VideoDecoderConfig,\n ) {}\n\n #ts = 0;\n #curAborter = { abort: false, st: performance.now() };\n find = async (time: number): Promise => {\n if (\n this.#dec == null ||\n this.#dec.state === 'closed' ||\n time <= this.#ts ||\n time - this.#ts > 3e6\n ) {\n this.#reset(time);\n }\n\n this.#curAborter.abort = true;\n this.#ts = time;\n\n this.#curAborter = { abort: false, st: performance.now() };\n const vf = await this.#parseFrame(time, this.#dec, this.#curAborter);\n this.#sleepCnt = 0;\n return vf;\n };\n\n // fix VideoFrame duration is null\n #lastVfDur = 0;\n\n #downgradeSoftDecode = false;\n #videoDecCusorIdx = 0;\n #videoFrames: VideoFrame[] = [];\n #outputFrameCnt = 0;\n #inputChunkCnt = 0;\n #sleepCnt = 0;\n #predecodeErr = false;\n #parseFrame = async (\n time: number,\n dec: VideoDecoder | null,\n aborter: { abort: boolean; st: number },\n ): Promise => {\n if (dec == null || dec.state === 'closed' || aborter.abort) return null;\n\n if (this.#videoFrames.length > 0) {\n const vf = this.#videoFrames[0];\n if (time < vf.timestamp) return null;\n // 弹出第一帧\n this.#videoFrames.shift();\n // 第一帧过期，找下一帧\n if (time > vf.timestamp + (vf.duration ?? 0)) {\n vf.close();\n return await this.#parseFrame(time, dec, aborter);\n }\n\n if (!this.#predecodeErr && this.#videoFrames.length < 10) {\n // 预解码 避免等待\n this.#startDecode(dec).catch((err) => {\n this.#predecodeErr = true;\n this.#reset(time);\n throw err;\n });\n }\n // 符合期望\n return vf;\n }\n\n // 缺少帧数据\n if (\n this.#decoding ||\n (this.#outputFrameCnt < this.#inputChunkCnt && dec.decodeQueueSize > 0)\n ) {\n if (performance.now() - aborter.st > 6e3) {\n throw Error(\n `MP4Clip.tick video timeout, ${JSON.stringify(this.#getState())}`,\n );\n }\n // 解码中，等待，然后重试\n this.#sleepCnt += 1;\n await sleep(15);\n } else if (this.#videoDecCusorIdx >= this.samples.length) {\n // decode completed\n return null;\n } else {\n try {\n await this.#startDecode(dec);\n } catch (err) {\n this.#reset(time);\n throw err;\n }\n }\n return await this.#parseFrame(time, dec, aborter);\n };\n\n #decoding = false;\n #startDecode = async (dec: VideoDecoder) => {\n if (this.#decoding || dec.decodeQueueSize > 600) return;\n\n // 启动解码任务，然后重试\n let endIdx = this.#videoDecCusorIdx + 1;\n if (endIdx > this.samples.length) return;\n\n this.#decoding = true;\n // 该 GoP 时间区间有时间匹配，且未被删除的帧\n let hasValidFrame = false;\n for (; endIdx < this.samples.length; endIdx++) {\n const s = this.samples[endIdx];\n if (!hasValidFrame && !s.deleted) {\n hasValidFrame = true;\n }\n // 找一个 GoP，所以是下一个 IDR 帧结束\n if (s.is_idr) break;\n }\n\n if (hasValidFrame) {\n const samples = this.samples.slice(this.#videoDecCusorIdx, endIdx);\n if (samples[0]?.is_idr !== true) {\n Log.warn('First sample not idr frame');\n } else {\n const readStarTime = performance.now();\n const chunks = await videosamples2Chunks(samples, this.localFileReader);\n\n const readCost = performance.now() - readStarTime;\n if (readCost > 1000) {\n const first = samples[0];\n const last = samples.at(-1)!;\n const rangSize = last.offset + last.size - first.offset;\n Log.warn(\n `Read video samples time cost: ${Math.round(readCost)}ms, file chunk size: ${rangSize}`,\n );\n }\n // Wait for the previous asynchronous operation to complete, at which point the task may have already been terminated\n if (dec.state === 'closed') return;\n\n this.#lastVfDur = chunks[0]?.duration ?? 0;\n decodeGoP(dec, chunks, {\n onDecodingError: (err) => {\n if (this.#downgradeSoftDecode) {\n throw err;\n } else if (this.#outputFrameCnt === 0) {\n this.#downgradeSoftDecode = true;\n Log.warn('Downgrade to software decode');\n this.#reset();\n }\n },\n });\n\n this.#inputChunkCnt += chunks.length;\n }\n }\n this.#videoDecCusorIdx = endIdx;\n this.#decoding = false;\n };\n\n #reset = (time?: number) => {\n this.#decoding = false;\n this.#videoFrames.forEach((f) => f.close());\n this.#videoFrames = [];\n if (time == null || time === 0) {\n this.#videoDecCusorIdx = 0;\n } else {\n let keyIdx = 0;\n for (let i = 0; i < this.samples.length; i++) {\n const s = this.samples[i];\n if (s.is_idr) keyIdx = i;\n if (s.cts < time) continue;\n this.#videoDecCusorIdx = keyIdx;\n break;\n }\n }\n this.#inputChunkCnt = 0;\n this.#outputFrameCnt = 0;\n if (this.#dec?.state !== 'closed') this.#dec?.close();\n const encoderConf = {\n ...this.conf,\n ...(this.#downgradeSoftDecode\n ? { hardwareAcceleration: 'prefer-software' }\n : {}),\n } as VideoDecoderConfig;\n this.#dec = new VideoDecoder({\n output: (vf) => {\n this.#outputFrameCnt += 1;\n if (vf.timestamp === -1) {\n vf.close();\n return;\n }\n let rsVf = vf;\n if (vf.duration == null) {\n rsVf = new VideoFrame(vf, {\n duration: this.#lastVfDur,\n });\n vf.close();\n }\n this.#videoFrames.push(rsVf);\n },\n error: (err) => {\n if (err.message.includes('Codec reclaimed due to inactivity')) {\n // todo: 因无活动被自动关闭的解码器，是否需要自动重启？\n this.#dec = null;\n Log.warn(err.message);\n return;\n }\n\n const errMsg = `VideoFinder VideoDecoder err: ${err.message}, config: ${JSON.stringify(encoderConf)}, state: ${JSON.stringify(this.#getState())}`;\n Log.error(errMsg);\n throw Error(errMsg);\n },\n });\n this.#dec.configure(encoderConf);\n };\n\n #getState = () => ({\n time: this.#ts,\n decState: this.#dec?.state,\n decQSize: this.#dec?.decodeQueueSize,\n decCusorIdx: this.#videoDecCusorIdx,\n sampleLen: this.samples.length,\n inputCnt: this.#inputChunkCnt,\n outputCnt: this.#outputFrameCnt,\n cacheFrameLen: this.#videoFrames.length,\n softDeocde: this.#downgradeSoftDecode,\n clipIdCnt: CLIP_ID,\n sleepCnt: this.#sleepCnt,\n memInfo: memoryUsageInfo(),\n });\n\n destroy = () => {\n if (this.#dec?.state !== 'closed') this.#dec?.close();\n this.#dec = null;\n this.#curAborter.abort = true;\n this.#videoFrames.forEach((f) => f.close());\n this.#videoFrames = [];\n this.localFileReader.close();\n };\n}\n\nfunction findIndexOfSamples(time: number, samples: ExtMP4Sample[]) {\n for (let i = 0; i < samples.length; i++) {\n const s = samples[i];\n if (time >= s.cts && time < s.cts + s.duration) {\n return i;\n }\n if (s.cts > time) break;\n }\n return 0;\n}\n\nclass AudioFrameFinder {\n #volume = 1;\n #sampleRate;\n constructor(\n public localFileReader: LocalFileReader,\n public samples: ExtMP4Sample[],\n public conf: AudioDecoderConfig,\n opts: { volume: number; targetSampleRate: number },\n ) {\n this.#volume = opts.volume;\n this.#sampleRate = opts.targetSampleRate;\n }\n\n #dec: ReturnType | null = null;\n #curAborter = { abort: false, st: performance.now() };\n find = async (time: number): Promise => {\n const needResetTime = time <= this.#ts || time - this.#ts > 0.1e6;\n if (this.#dec == null || this.#dec.state === 'closed' || needResetTime) {\n this.#reset();\n }\n\n if (needResetTime) {\n // 前后获取音频数据差异不能超过 100ms(经验值)，否则视为 seek 操作，重置解码器\n // seek 操作，重置时间\n this.#ts = time;\n this.#decCusorIdx = findIndexOfSamples(time, this.samples);\n }\n\n this.#curAborter.abort = true;\n const deltaTime = time - this.#ts;\n this.#ts = time;\n\n this.#curAborter = { abort: false, st: performance.now() };\n\n const pcmData = await this.#parseFrame(\n Math.ceil(deltaTime * (this.#sampleRate / 1e6)),\n this.#dec,\n this.#curAborter,\n );\n this.#sleepCnt = 0;\n return pcmData;\n };\n\n #ts = 0;\n #decCusorIdx = 0;\n #pcmData: {\n frameCnt: number;\n data: [Float32Array, Float32Array][];\n } = {\n frameCnt: 0,\n data: [],\n };\n #sleepCnt = 0;\n #parseFrame = async (\n emitFrameCnt: number,\n dec: ReturnType | null = null,\n aborter: { abort: boolean; st: number },\n ): Promise => {\n if (\n dec == null ||\n aborter.abort ||\n dec.state === 'closed' ||\n emitFrameCnt === 0\n ) {\n return [];\n }\n\n // 数据满足需要\n const ramainFrameCnt = this.#pcmData.frameCnt - emitFrameCnt;\n if (ramainFrameCnt > 0) {\n // 剩余音频数据小于 100ms，预先解码\n if (ramainFrameCnt < DEFAULT_AUDIO_CONF.sampleRate / 10) {\n this.#startDecode(dec);\n }\n return emitAudioFrames(this.#pcmData, emitFrameCnt);\n }\n\n if (dec.decoding) {\n if (performance.now() - aborter.st > 3e3) {\n aborter.abort = true;\n throw Error(\n `MP4Clip.tick audio timeout, ${JSON.stringify(this.#getState())}`,\n );\n }\n // 解码中，等待\n this.#sleepCnt += 1;\n await sleep(15);\n } else if (this.#decCusorIdx >= this.samples.length - 1) {\n // 最后片段，返回剩余数据\n return emitAudioFrames(this.#pcmData, this.#pcmData.frameCnt);\n } else {\n this.#startDecode(dec);\n }\n return this.#parseFrame(emitFrameCnt, dec, aborter);\n };\n\n #startDecode = (dec: ReturnType) => {\n const onceDecodeCnt = 10;\n if (dec.decodeQueueSize > onceDecodeCnt) return;\n // 启动解码任务\n const samples = [];\n let i = this.#decCusorIdx;\n while (i < this.samples.length) {\n const s = this.samples[i];\n i += 1;\n if (s.deleted) continue;\n samples.push(s);\n if (samples.length >= onceDecodeCnt) break;\n }\n this.#decCusorIdx = i;\n\n dec.decode(\n samples.map(\n (s) =>\n new EncodedAudioChunk({\n type: 'key',\n timestamp: s.cts,\n duration: s.duration,\n data: s.data!,\n }),\n ),\n );\n };\n\n #reset = () => {\n this.#ts = 0;\n this.#decCusorIdx = 0;\n this.#pcmData = {\n frameCnt: 0,\n data: [],\n };\n this.#dec?.close();\n this.#dec = createAudioChunksDecoder(\n this.conf,\n {\n resampleRate: DEFAULT_AUDIO_CONF.sampleRate,\n volume: this.#volume,\n },\n (pcmArr) => {\n this.#pcmData.data.push(pcmArr as [Float32Array, Float32Array]);\n this.#pcmData.frameCnt += pcmArr[0].length;\n },\n );\n };\n\n #getState = () => ({\n time: this.#ts,\n decState: this.#dec?.state,\n decQSize: this.#dec?.decodeQueueSize,\n decCusorIdx: this.#decCusorIdx,\n sampleLen: this.samples.length,\n pcmLen: this.#pcmData.frameCnt,\n clipIdCnt: CLIP_ID,\n sleepCnt: this.#sleepCnt,\n memInfo: memoryUsageInfo(),\n });\n\n destroy = () => {\n this.#dec = null;\n this.#curAborter.abort = true;\n this.#pcmData = {\n frameCnt: 0,\n data: [],\n };\n this.localFileReader.close();\n };\n}\n\nfunction createAudioChunksDecoder(\n decoderConf: AudioDecoderConfig,\n opts: { resampleRate: number; volume: number },\n outputCb: (pcm: Float32Array[]) => void,\n) {\n let inputCnt = 0;\n let outputCnt = 0;\n const outputHandler = (pcmArr: Float32Array[]) => {\n outputCnt += 1;\n if (pcmArr.length === 0) return;\n // 音量调节\n if (opts.volume !== 1) {\n for (const pcm of pcmArr)\n for (let i = 0; i < pcm.length; i++) pcm[i] *= opts.volume;\n }\n\n // 补齐双声道\n if (pcmArr.length === 1) pcmArr = [pcmArr[0], pcmArr[0]];\n\n outputCb(pcmArr);\n };\n const resampleQ = createPromiseQueue(outputHandler);\n\n const needResample = opts.resampleRate !== decoderConf.sampleRate;\n let adec = new AudioDecoder({\n output: (ad) => {\n const pcm = extractPCM4AudioData(ad);\n if (needResample) {\n resampleQ(() =>\n audioResample(pcm, ad.sampleRate, {\n rate: opts.resampleRate,\n chanCount: ad.numberOfChannels,\n }),\n );\n } else {\n outputHandler(pcm);\n }\n ad.close();\n },\n error: (err) => {\n if (err.message.includes('Codec reclaimed due to inactivity')) {\n return;\n }\n handleDecodeError('MP4Clip AudioDecoder err', err as Error);\n },\n });\n adec.configure(decoderConf);\n\n function handleDecodeError(prefixStr: string, err: Error) {\n const errMsg = `${prefixStr}: ${(err as Error).message}, state: ${JSON.stringify(\n {\n qSize: adec.decodeQueueSize,\n state: adec.state,\n inputCnt,\n outputCnt,\n },\n )}`;\n Log.error(errMsg);\n throw Error(errMsg);\n }\n\n return {\n decode(chunks: EncodedAudioChunk[]) {\n inputCnt += chunks.length;\n try {\n for (const chunk of chunks) adec.decode(chunk);\n } catch (err) {\n handleDecodeError('decode audio chunk error', err as Error);\n }\n },\n close() {\n if (adec.state !== 'closed') adec.close();\n },\n get decoding() {\n return inputCnt > outputCnt && adec.decodeQueueSize > 0;\n },\n get state() {\n return adec.state;\n },\n get decodeQueueSize() {\n return adec.decodeQueueSize;\n },\n };\n}\n\n// 并行执行任务，但按顺序emit结果\nfunction createPromiseQueue(onResult: (data: T) => void) {\n const rsCache: T[] = [];\n let waitingIdx = 0;\n\n function updateRs(rs: T, emitIdx: number) {\n rsCache[emitIdx] = rs;\n emitRs();\n }\n\n function emitRs() {\n const rs = rsCache[waitingIdx];\n if (rs == null) return;\n onResult(rs);\n\n waitingIdx += 1;\n emitRs();\n }\n\n let addIdx = 0;\n return (task: () => Promise) => {\n const emitIdx = addIdx;\n addIdx += 1;\n task()\n .then((rs) => updateRs(rs, emitIdx))\n .catch((err) => updateRs(err, emitIdx));\n };\n}\n\nfunction emitAudioFrames(\n pcmData: { frameCnt: number; data: [Float32Array, Float32Array][] },\n emitCnt: number,\n) {\n // todo: perf 重复利用内存空间\n const audio = [new Float32Array(emitCnt), new Float32Array(emitCnt)];\n let offset = 0;\n let i = 0;\n for (; i < pcmData.data.length; ) {\n const [chan0, chan1] = pcmData.data[i];\n if (offset + chan0.length > emitCnt) {\n const gapCnt = emitCnt - offset;\n audio[0].set(chan0.subarray(0, gapCnt), offset);\n audio[1].set(chan1.subarray(0, gapCnt), offset);\n pcmData.data[i][0] = chan0.subarray(gapCnt, chan0.length);\n pcmData.data[i][1] = chan1.subarray(gapCnt, chan1.length);\n break;\n } else {\n audio[0].set(chan0, offset);\n audio[1].set(chan1, offset);\n offset += chan0.length;\n i++;\n }\n }\n pcmData.data = pcmData.data.slice(i);\n pcmData.frameCnt -= emitCnt;\n return audio;\n}\n\nasync function videosamples2Chunks(\n samples: ExtMP4Sample[],\n reader: Awaited>,\n): Promise {\n const first = samples[0];\n const last = samples.at(-1);\n if (last == null) return [];\n\n const rangSize = last.offset + last.size - first.offset;\n if (rangSize < 30e6) {\n // 单次读取数据小于 30M，就一次性读取数据，降低 IO 频次\n const data = new Uint8Array(\n await reader.read(rangSize, { at: first.offset }),\n );\n return samples.map((s) => {\n const offset = s.offset - first.offset;\n return new EncodedVideoChunk({\n type: s.is_sync ? 'key' : 'delta',\n timestamp: s.cts,\n duration: s.duration,\n data: data.subarray(offset, offset + s.size),\n });\n });\n }\n\n return await Promise.all(\n samples.map(async (s) => {\n return new EncodedVideoChunk({\n type: s.is_sync ? 'key' : 'delta',\n timestamp: s.cts,\n duration: s.duration,\n data: await reader.read(s.size, {\n at: s.offset,\n }),\n });\n }),\n );\n}\n\nfunction createVF2BlobConvtr(\n width: number,\n height: number,\n opts?: ImageEncodeOptions,\n) {\n const cvs = new OffscreenCanvas(width, height);\n const ctx = cvs.getContext('2d')!;\n\n return async (vf: VideoFrame) => {\n ctx.drawImage(vf, 0, 0, width, height);\n vf.close();\n const blob = await cvs.convertToBlob(opts);\n return blob;\n };\n}\n\nfunction splitVideoSampleByTime(videoSamples: ExtMP4Sample[], time: number) {\n if (videoSamples.length === 0) return [];\n let gopStartIdx = 0;\n let gopEndIdx = 0;\n let hitIdx = -1;\n for (let i = 0; i < videoSamples.length; i++) {\n const s = videoSamples[i];\n if (hitIdx === -1 && time < s.cts) hitIdx = i - 1;\n if (s.is_idr) {\n if (hitIdx === -1) {\n gopStartIdx = i;\n } else {\n gopEndIdx = i;\n break;\n }\n }\n }\n\n const hitSample = videoSamples[hitIdx];\n if (hitSample == null) throw Error('Not found video sample by time');\n\n const preSlice = videoSamples\n .slice(0, gopEndIdx === 0 ? videoSamples.length : gopEndIdx)\n .map((s) => ({ ...s }));\n for (let i = gopStartIdx; i < preSlice.length; i++) {\n const s = preSlice[i];\n if (time < s.cts) {\n s.deleted = true;\n s.cts = -1;\n }\n }\n fixFirstBlackFrame(preSlice);\n\n const postSlice = videoSamples\n .slice(hitSample.is_idr ? hitIdx : gopStartIdx)\n .map((s) => ({ ...s, cts: s.cts - time }));\n\n for (const s of postSlice) {\n if (s.cts < 0) {\n s.deleted = true;\n s.cts = -1;\n }\n }\n fixFirstBlackFrame(postSlice);\n\n return [preSlice, postSlice];\n}\n\nfunction splitAudioSampleByTime(audioSamples: ExtMP4Sample[], time: number) {\n if (audioSamples.length === 0) return [];\n let hitIdx = -1;\n for (let i = 0; i < audioSamples.length; i++) {\n const s = audioSamples[i];\n if (time > s.cts) continue;\n hitIdx = i;\n break;\n }\n if (hitIdx === -1) throw Error('Not found audio sample by time');\n const preSlice = audioSamples.slice(0, hitIdx).map((s) => ({ ...s }));\n const postSlice = audioSamples\n .slice(hitIdx)\n .map((s) => ({ ...s, cts: s.cts - time }));\n return [preSlice, postSlice];\n}\n\n// 兼容解码错误\nfunction decodeGoP(\n dec: VideoDecoder,\n chunks: EncodedVideoChunk[],\n opts: {\n onDecodingError?: (err: Error) => void;\n },\n) {\n let i = 0;\n if (dec.state !== 'configured') return;\n for (; i < chunks.length; i++) dec.decode(chunks[i]);\n\n // todo：flush 之后下一帧必须是 IDR 帧，是否可以根据情况再决定调用 flush？\n // windows 某些设备 flush 可能不会被 resolved，所以不能 await flush\n dec.flush().catch((err) => {\n if (!(err instanceof Error)) throw err;\n if (\n err.message.includes('Decoding error') &&\n opts.onDecodingError != null\n ) {\n opts.onDecodingError(err);\n return;\n }\n // reset 中断解码器，预期会抛出 AbortedError\n if (!err.message.includes('Aborted due to close')) {\n throw err;\n }\n });\n}\n\nfunction idrNALUOffset(\n u8Arr: Uint8Array,\n type: MP4Sample['description']['type'],\n) {\n if (type !== 'avc1' && type !== 'hvc1') return 0;\n\n const dv = new DataView(u8Arr.buffer);\n let i = 0;\n for (; i < u8Arr.byteLength - 4; ) {\n if (type === 'avc1' && (dv.getUint8(i + 4) & 0x1f) === 5) {\n return i;\n } else if (type === 'hvc1') {\n const nalUnitType = (dv.getUint8(i + 4) >> 1) & 0x3f;\n if (nalUnitType === 19 || nalUnitType === 20) return i;\n }\n // 跳至下一个 NALU 继续检查\n i += dv.getUint32(i) + 4;\n }\n return -1;\n}\n\nasync function thumbnailByKeyFrame(\n samples: ExtMP4Sample[],\n localFile: OPFSToolFile,\n decConf: VideoDecoderConfig,\n abortSingl: AbortSignal,\n time: { start: number; end: number },\n onOutput: (vf: VideoFrame | null, done: boolean) => void,\n) {\n const fileReader = await localFile.createReader();\n\n const chunks = await videosamples2Chunks(\n samples.filter(\n (s) =>\n !s.deleted && s.is_sync && s.cts >= time.start && s.cts <= time.end,\n ),\n fileReader,\n );\n if (chunks.length === 0 || abortSingl.aborted) return;\n\n let outputCnt = 0;\n decodeGoP(createVideoDec(), chunks, {\n onDecodingError: (err) => {\n Log.warn('thumbnailsByKeyFrame', err);\n // 尝试降级一次\n if (outputCnt === 0) {\n decodeGoP(createVideoDec(true), chunks, {\n onDecodingError: (err) => {\n fileReader.close();\n Log.error('thumbnailsByKeyFrame retry soft deocde', err);\n },\n });\n } else {\n onOutput(null, true);\n fileReader.close();\n }\n },\n });\n\n function createVideoDec(downgrade = false) {\n const encoderConf = {\n ...decConf,\n ...(downgrade ? { hardwareAcceleration: 'prefer-software' } : {}),\n } as VideoDecoderConfig;\n const dec = new VideoDecoder({\n output: (vf) => {\n outputCnt += 1;\n const done = outputCnt === chunks.length;\n onOutput(vf, done);\n if (done) {\n fileReader.close();\n if (dec.state !== 'closed') dec.close();\n }\n },\n error: (err) => {\n const errMsg = `thumbnails decoder error: ${err.message}, config: ${JSON.stringify(encoderConf)}, state: ${JSON.stringify(\n {\n qSize: dec.decodeQueueSize,\n state: dec.state,\n outputCnt,\n inputCnt: chunks.length,\n },\n )}`;\n Log.error(errMsg);\n throw Error(errMsg);\n },\n });\n abortSingl.addEventListener('abort', () => {\n fileReader.close();\n if (dec.state !== 'closed') dec.close();\n });\n dec.configure(encoderConf);\n return dec;\n }\n}\n\n// 如果第一帧出现的时间偏移较大，会导致第一帧为黑帧，这里尝试自动消除第一帧前的黑帧\nfunction fixFirstBlackFrame(samples: ExtMP4Sample[]) {\n let iframeCnt = 0;\n let minCtsSample: ExtMP4Sample | null = null;\n // cts 最小表示视频的第一帧\n for (const s of samples) {\n if (s.deleted) continue;\n // 最多检测两个 I 帧之间的帧\n if (s.is_sync) iframeCnt += 1;\n if (iframeCnt >= 2) break;\n\n if (minCtsSample == null || s.cts < minCtsSample.cts) {\n minCtsSample = s;\n }\n }\n // 200ms 是经验值，自动消除 200ms 内的黑帧，超过则不处理\n if (minCtsSample != null && minCtsSample.cts < 200e3) {\n minCtsSample.duration += minCtsSample.cts;\n minCtsSample.cts = 0;\n }\n}\n\nfunction memoryUsageInfo() {\n try {\n // @ts-ignore\n const mem = performance.memory;\n return {\n jsHeapSizeLimit: mem.jsHeapSizeLimit,\n totalJSHeapSize: mem.totalJSHeapSize,\n usedJSHeapSize: mem.usedJSHeapSize,\n percentUsed: (mem.usedJSHeapSize / mem.jsHeapSizeLimit).toFixed(3),\n percentTotal: (mem.totalJSHeapSize / mem.jsHeapSizeLimit).toFixed(3),\n };\n } catch (err) {\n return {};\n }\n}\n","import { Log } from '@webrock/internal-utils';\nimport { decodeImg } from '../av-utils';\nimport { IClip } from './iclip';\n\ntype AnimateImgType = 'avif' | 'webp' | 'png' | 'gif';\n\n/**\n * 图像素材，支持动图\n *\n * 普通文字可通过 {@link renderTxt2ImgBitmap} 转换成图片素材\n *\n * @example\n * new ImgClip((await fetch('')).body);\n *\n * @example\n * new ImgClip(\n * await renderTxt2ImgBitmap(\n * '水印',\n * `font-size:40px; color: white; text-shadow: 2px 2px 6px red;`,\n * )\n * )\n *\n * @see [视频合成](https://webav-tech.github.io/WebAV/demo/2_1-concat-video)\n */\nexport class ImgClip implements IClip {\n ready: IClip['ready'];\n\n #meta = {\n // 微秒\n duration: 0,\n width: 0,\n height: 0,\n };\n\n /**\n * ⚠️ 静态图片的 duration 为 Infinity\n *\n * 使用 Sprite 包装时需要将它的 duration 设置为有限数\n *\n */\n get meta() {\n return { ...this.#meta };\n }\n\n #img: ImageBitmap | null = null;\n\n #frames: VideoFrame[] = [];\n\n /**\n * 静态图片可使用流、ImageBitmap 初始化\n *\n * 动图需要使用 VideoFrame[] 或提供图片类型\n */\n constructor(\n dataSource:\n | ReadableStream\n | ImageBitmap\n | VideoFrame[]\n | { type: `image/${AnimateImgType}`; stream: ReadableStream },\n ) {\n const initWithImgBitmap = (imgBitmap: ImageBitmap) => {\n this.#img = imgBitmap;\n this.#meta.width = imgBitmap.width;\n this.#meta.height = imgBitmap.height;\n this.#meta.duration = Infinity;\n return { ...this.#meta };\n };\n\n if (dataSource instanceof ReadableStream) {\n this.ready = new Response(dataSource)\n .blob()\n .then((data) => createImageBitmap(data))\n .then(initWithImgBitmap);\n } else if (dataSource instanceof ImageBitmap) {\n this.ready = Promise.resolve(initWithImgBitmap(dataSource));\n } else if (\n Array.isArray(dataSource) &&\n dataSource.every((it) => it instanceof VideoFrame)\n ) {\n this.#frames = dataSource;\n const frame = this.#frames[0];\n if (frame == null) throw Error('The frame count must be greater than 0');\n this.#meta = {\n width: frame.displayWidth,\n height: frame.displayHeight,\n duration: this.#frames.reduce(\n (acc, cur) => acc + (cur.duration ?? 0),\n 0,\n ),\n };\n this.ready = Promise.resolve({ ...this.#meta, duration: Infinity });\n } else if ('type' in dataSource) {\n this.ready = this.#initAnimateImg(\n dataSource.stream,\n dataSource.type,\n ).then(() => ({\n width: this.#meta.width,\n height: this.#meta.height,\n duration: Infinity,\n }));\n } else {\n throw Error('Illegal arguments');\n }\n }\n\n async #initAnimateImg(\n stream: ReadableStream,\n type: `image/${AnimateImgType}`,\n ) {\n this.#frames = await decodeImg(stream, type);\n const firstVf = this.#frames[0];\n if (firstVf == null) throw Error('No frame available in gif');\n\n this.#meta = {\n duration: this.#frames.reduce((acc, cur) => acc + (cur.duration ?? 0), 0),\n width: firstVf.codedWidth,\n height: firstVf.codedHeight,\n };\n Log.info('ImgClip ready:', this.#meta);\n }\n\n tickInterceptor: >>(\n time: number,\n tickRet: T,\n ) => Promise = async (_, tickRet) => tickRet;\n\n async tick(time: number): Promise<{\n video: ImageBitmap | VideoFrame;\n state: 'success';\n }> {\n if (this.#img != null) {\n return await this.tickInterceptor(time, {\n video: await createImageBitmap(this.#img),\n state: 'success',\n });\n }\n const tt = time % this.#meta.duration;\n return await this.tickInterceptor(time, {\n video: (\n this.#frames.find(\n (f) => tt >= f.timestamp && tt <= f.timestamp + (f.duration ?? 0),\n ) ?? this.#frames[0]\n ).clone(),\n state: 'success',\n });\n }\n\n async split(time: number) {\n await this.ready;\n if (this.#img != null) {\n return [\n new ImgClip(await createImageBitmap(this.#img)),\n new ImgClip(await createImageBitmap(this.#img)),\n ] as [this, this];\n }\n let hitIdx = -1;\n for (let i = 0; i < this.#frames.length; i++) {\n const vf = this.#frames[i];\n if (time > vf.timestamp) continue;\n hitIdx = i;\n break;\n }\n if (hitIdx === -1) throw Error('Not found frame by time');\n const preSlice = this.#frames\n .slice(0, hitIdx)\n .map((vf) => new VideoFrame(vf));\n const postSlice = this.#frames.slice(hitIdx).map(\n (vf) =>\n new VideoFrame(vf, {\n timestamp: vf.timestamp - time,\n }),\n );\n return [new ImgClip(preSlice), new ImgClip(postSlice)] as [this, this];\n }\n\n async clone() {\n await this.ready;\n const data =\n this.#img == null\n ? this.#frames.map((vf) => vf.clone())\n : await createImageBitmap(this.#img);\n return new ImgClip(data) as this;\n }\n\n destroy(): void {\n Log.info('ImgClip destroy');\n this.#img?.close();\n this.#frames.forEach((f) => f.close());\n }\n}\n","import { Log } from '@webrock/internal-utils';\nimport {\n concatPCMFragments,\n extractPCM4AudioBuffer,\n ringSliceFloat32Array,\n} from '../av-utils';\nimport { DEFAULT_AUDIO_CONF, IClip } from './iclip';\n\ninterface IAudioClipOpts {\n loop?: boolean;\n volume?: number;\n}\n\n/**\n * 音频素材，为创建、编辑音视频功能提供音频数据\n *\n * @example\n * new AudioClip((await fetch('')).body, {\n * loop: true,\n * }),\n */\nexport class AudioClip implements IClip {\n static ctx: AudioContext | null = null;\n\n ready: IClip['ready'];\n\n #meta = {\n // 微秒\n duration: 0,\n width: 0,\n height: 0,\n };\n\n /**\n * 音频元信息\n *\n * ⚠️ 注意，这里是转换后（标准化）的元信息，非原始音频元信息\n */\n get meta() {\n return {\n ...this.#meta,\n sampleRate: DEFAULT_AUDIO_CONF.sampleRate,\n chanCount: 2,\n };\n }\n\n #chan0Buf: Float32Array = new Float32Array();\n #chan1Buf: Float32Array = new Float32Array();\n /**\n * 获取音频素材完整的 PCM 数据\n */\n getPCMData(): Float32Array[] {\n return [this.#chan0Buf, this.#chan1Buf];\n }\n\n #opts;\n\n /**\n *\n * @param dataSource 音频文件流\n * @param opts 音频配置，控制音量、是否循环\n */\n constructor(\n dataSource: ReadableStream | Float32Array[],\n opts: IAudioClipOpts = {},\n ) {\n this.#opts = {\n loop: false,\n volume: 1,\n ...opts,\n };\n\n this.ready = this.#init(dataSource).then(() => ({\n // audio 没有宽高，无需绘制\n width: 0,\n height: 0,\n duration: opts.loop ? Infinity : this.#meta.duration,\n }));\n }\n\n async #init(\n dataSource: ReadableStream | Float32Array[],\n ): Promise {\n if (AudioClip.ctx == null) {\n AudioClip.ctx = new AudioContext({\n sampleRate: DEFAULT_AUDIO_CONF.sampleRate,\n });\n }\n\n const tStart = performance.now();\n const pcm =\n dataSource instanceof ReadableStream\n ? await parseStream2PCM(dataSource, AudioClip.ctx)\n : dataSource;\n\n Log.info('Audio clip decoded complete:', performance.now() - tStart);\n\n const volume = this.#opts.volume;\n if (volume !== 1) {\n for (const chan of pcm)\n for (let i = 0; i < chan.length; i += 1) chan[i] *= volume;\n }\n\n this.#meta.duration = (pcm[0].length / DEFAULT_AUDIO_CONF.sampleRate) * 1e6;\n\n this.#chan0Buf = pcm[0];\n // 单声道 转 立体声\n this.#chan1Buf = pcm[1] ?? this.#chan0Buf;\n\n Log.info(\n 'Audio clip convert to AudioData, time:',\n performance.now() - tStart,\n );\n }\n\n /**\n * 拦截 {@link AudioClip.tick} 方法返回的数据，用于对音频数据二次处理\n * @param time 调用 tick 的时间\n * @param tickRet tick 返回的数据\n *\n * @see [移除视频绿幕背景](https://webav-tech.github.io/WebAV/demo/3_2-chromakey-video)\n */\n tickInterceptor: >>(\n time: number,\n tickRet: T,\n ) => Promise = async (_, tickRet) => tickRet;\n\n // 微秒\n #ts = 0;\n #frameOffset = 0;\n /**\n * 返回上次与当前时刻差对应的音频 PCM 数据；\n *\n * 若差值超过 3s 或当前时间小于上次时间，则重置状态\n * @example\n * tick(0) // => []\n * tick(1e6) // => [leftChanPCM(1s), rightChanPCM(1s)]\n *\n */\n async tick(time: number): Promise<{\n audio: Float32Array[];\n state: 'success' | 'done';\n }> {\n if (!this.#opts.loop && time >= this.#meta.duration) {\n // 待观察：如果time跨度较大，返回done，理论上会丢失一些音频帧\n return await this.tickInterceptor(time, { audio: [], state: 'done' });\n }\n\n const deltaTime = time - this.#ts;\n\n // reset\n if (time < this.#ts || deltaTime > 3e6) {\n this.#ts = time;\n this.#frameOffset = Math.ceil(\n (this.#ts / 1e6) * DEFAULT_AUDIO_CONF.sampleRate,\n );\n return await this.tickInterceptor(time, {\n audio: [new Float32Array(0), new Float32Array(0)],\n state: 'success',\n });\n }\n\n this.#ts = time;\n const frameCnt = Math.ceil(\n (deltaTime / 1e6) * DEFAULT_AUDIO_CONF.sampleRate,\n );\n const endIdx = this.#frameOffset + frameCnt;\n const audio = this.#opts.loop\n ? [\n ringSliceFloat32Array(this.#chan0Buf, this.#frameOffset, endIdx),\n ringSliceFloat32Array(this.#chan1Buf, this.#frameOffset, endIdx),\n ]\n : [\n this.#chan0Buf.slice(this.#frameOffset, endIdx),\n this.#chan1Buf.slice(this.#frameOffset, endIdx),\n ];\n this.#frameOffset = endIdx;\n\n return await this.tickInterceptor(time, { audio, state: 'success' });\n }\n\n /**\n * 按指定时间切割，返回前后两个音频素材\n * @param time 时间，单位微秒\n */\n async split(time: number) {\n await this.ready;\n const frameCnt = Math.ceil((time / 1e6) * DEFAULT_AUDIO_CONF.sampleRate);\n const preSlice = new AudioClip(\n this.getPCMData().map((chan) => chan.slice(0, frameCnt)),\n this.#opts,\n );\n const postSlice = new AudioClip(\n this.getPCMData().map((chan) => chan.slice(frameCnt)),\n this.#opts,\n );\n return [preSlice, postSlice] as [this, this];\n }\n\n async clone() {\n await this.ready;\n const clip = new AudioClip(this.getPCMData(), this.#opts) as this;\n await clip.ready;\n return clip;\n }\n\n /**\n * 销毁实例，释放资源\n */\n destroy(): void {\n this.#chan0Buf = new Float32Array(0);\n this.#chan1Buf = new Float32Array(0);\n Log.info('---- audioclip destroy ----');\n }\n}\n\n/**\n * 拼接多个 AudioClip\n */\nexport async function concatAudioClip(\n clips: AudioClip[],\n opts?: IAudioClipOpts,\n) {\n const bufs: Float32Array[][] = [];\n for (const clip of clips) {\n await clip.ready;\n bufs.push(clip.getPCMData());\n }\n return new AudioClip(concatPCMFragments(bufs), opts);\n}\n\nasync function parseStream2PCM(\n stream: ReadableStream,\n ctx: AudioContext | OfflineAudioContext,\n): Promise {\n const buf = await new Response(stream).arrayBuffer();\n return extractPCM4AudioBuffer(await ctx.decodeAudioData(buf));\n}\n","import { autoReadStream } from '@webrock/internal-utils';\nimport { IClip } from './iclip';\n\n/**\n * 包装实时音视频流，仅用于 [AVCanvas](../../av-canvas/classes/AVCanvas.html)\n *\n * ⚠️ 不可用于 {@link Combinator} ，因为后台合成视频的速度是快于物理时间的，实时流无法提供非实时的数据\n *\n * @example\n * const spr = new VisibleSprite(\n * new MediaStreamClip(\n * await navigator.mediaDevices.getUserMedia({ video: true, audio: true, }),\n * ),\n * );\n * await avCvs.addSprite(spr);\n */\nexport class MediaStreamClip implements IClip {\n static ctx: AudioContext | null = null;\n\n ready: IClip['ready'];\n\n #meta = {\n // 微秒\n duration: 0,\n width: 0,\n height: 0,\n };\n\n get meta() {\n return {\n ...this.#meta,\n };\n }\n\n #stopRenderCvs = () => {};\n\n /**\n * 实时流的音轨\n */\n readonly audioTrack: MediaStreamAudioTrack | null;\n\n #cvs: OffscreenCanvas | null = null;\n\n #ms: MediaStream;\n constructor(ms: MediaStream) {\n this.#ms = ms;\n this.audioTrack = ms.getAudioTracks()[0] ?? null;\n this.#meta.duration = Infinity;\n const videoTrack = ms.getVideoTracks()[0];\n if (videoTrack != null) {\n videoTrack.contentHint = 'motion';\n this.ready = new Promise((resolve) => {\n this.#stopRenderCvs = renderVideoTrackToCvs(videoTrack, (cvs) => {\n this.#meta.width = cvs.width;\n this.#meta.height = cvs.height;\n this.#cvs = cvs;\n resolve(this.meta);\n });\n });\n } else {\n this.ready = Promise.resolve(this.meta);\n }\n }\n\n async tick(): Promise<{\n video: ImageBitmap | null;\n audio: Float32Array[];\n state: 'success';\n }> {\n return {\n video: this.#cvs == null ? null : await createImageBitmap(this.#cvs),\n audio: [],\n state: 'success',\n };\n }\n\n async split() {\n return [await this.clone(), await this.clone()] as [this, this];\n }\n\n async clone() {\n return new MediaStreamClip(this.#ms.clone()) as this;\n }\n\n destroy(): void {\n this.#ms.getTracks().forEach((t) => t.stop());\n this.#stopRenderCvs();\n }\n}\n\nfunction renderVideoTrackToCvs(\n track: MediaStreamVideoTrack,\n onOffscreenCanvasReady: (cvs: OffscreenCanvas) => void,\n) {\n let emitFF = false;\n let cvsCtx: OffscreenCanvasRenderingContext2D;\n return autoReadStream(\n new MediaStreamTrackProcessor({\n track,\n }).readable,\n {\n onChunk: async (frame) => {\n if (!emitFF) {\n const { displayHeight, displayWidth } = frame;\n const width = displayWidth ?? 0;\n const height = displayHeight ?? 0;\n const cvs = new OffscreenCanvas(width, height);\n cvsCtx = cvs.getContext('2d')!;\n onOffscreenCanvasReady(cvs);\n emitFF = true;\n }\n cvsCtx.drawImage(frame, 0, 0);\n frame.close();\n },\n onDone: async () => {},\n },\n );\n}\n","import { IClip } from './iclip';\r\n\r\ninterface IEmbedSubtitlesOpts {\r\n color?: string;\r\n textBgColor?: string | null;\r\n type?: 'srt';\r\n fontFamily?: string;\r\n fontSize?: number;\r\n letterSpacing?: string | null;\r\n // 字幕偏离底部的距离\r\n bottomOffset?: number;\r\n strokeStyle?: string;\r\n lineWidth?: number | null;\r\n lineCap?: CanvasLineCap | null;\r\n lineJoin?: CanvasLineJoin | null;\r\n textShadow?: {\r\n offsetX: number;\r\n offsetY: number;\r\n blur: number;\r\n color: string;\r\n };\r\n videoWidth: number;\r\n videoHeight: number;\r\n fontWeight?: string | number;\r\n fontStyle?: 'normal' | 'italic';\r\n}\r\n\r\ndeclare global {\r\n interface OffscreenCanvasRenderingContext2D {\r\n letterSpacing: string;\r\n }\r\n}\r\n\r\ninterface SubtitleStruct {\r\n start: number;\r\n end: number;\r\n text: string;\r\n}\r\n\r\n/**\r\n * 嵌入式字幕，将字幕（目前仅支持 SRT 格式）嵌入视频画面中\r\n *\r\n * @example\r\n * const es = new EmbedSubtitlesClip(srtSubtitleStr, {\r\n * videoWidth: 1280,\r\n * videoHeight: 720,\r\n * fontFamily: 'Noto Sans SC',\r\n * color: 'white',\r\n * });\r\n */\r\nexport class EmbedSubtitlesClip implements IClip {\r\n ready: IClip['ready'];\r\n\r\n #subtitles: SubtitleStruct[] = [];\r\n\r\n #meta = {\r\n width: 0,\r\n height: 0,\r\n duration: 0,\r\n };\r\n\r\n get meta() {\r\n return { ...this.#meta };\r\n }\r\n\r\n #opts: Required = {\r\n color: '#FFF',\r\n textBgColor: null,\r\n type: 'srt',\r\n fontSize: 30,\r\n letterSpacing: null,\r\n bottomOffset: 30,\r\n fontFamily: 'Noto Sans SC',\r\n strokeStyle: '#000',\r\n lineWidth: null,\r\n lineCap: null,\r\n lineJoin: null,\r\n textShadow: {\r\n offsetX: 2,\r\n offsetY: 2,\r\n blur: 4,\r\n color: '#000',\r\n },\r\n videoWidth: 1280,\r\n videoHeight: 720,\r\n fontWeight: 'normal',\r\n fontStyle: 'normal',\r\n };\r\n\r\n #cvs: OffscreenCanvas;\r\n #ctx: OffscreenCanvasRenderingContext2D;\r\n\r\n #lastVF: VideoFrame | null = null;\r\n\r\n #lineHeight = 0;\r\n #linePadding = 0;\r\n\r\n constructor(content: string | SubtitleStruct[], opts: IEmbedSubtitlesOpts) {\r\n this.#subtitles = Array.isArray(content)\r\n ? content\r\n : parseSrt(content).map(({ start, end, text }) => ({\r\n start: start * 1e6,\r\n end: end * 1e6,\r\n text,\r\n }));\r\n if (this.#subtitles.length === 0) throw Error('No subtitles content');\r\n\r\n this.#opts = Object.assign(this.#opts, opts);\r\n // 如果需要绘制背景，则需要给文字添加边距\r\n this.#linePadding =\r\n opts.textBgColor == null ? 0 : (opts.fontSize ?? 50) * 0.2;\r\n\r\n const {\r\n fontSize,\r\n fontFamily,\r\n fontWeight,\r\n fontStyle,\r\n videoWidth,\r\n videoHeight,\r\n letterSpacing,\r\n } = this.#opts;\r\n this.#lineHeight = fontSize + this.#linePadding * 2;\r\n this.#cvs = new OffscreenCanvas(videoWidth, videoHeight);\r\n this.#ctx = this.#cvs.getContext('2d')!;\r\n this.#ctx.font = `${fontStyle} ${fontWeight} ${fontSize}px ${fontFamily}`;\r\n this.#ctx.textAlign = 'center';\r\n this.#ctx.textBaseline = 'top';\r\n this.#ctx.letterSpacing = letterSpacing ?? '0px';\r\n\r\n this.#meta = {\r\n width: videoWidth,\r\n height: videoHeight,\r\n duration: this.#subtitles.at(-1)?.end ?? 0,\r\n };\r\n // 字幕的宽高 由视频画面内容决定\r\n this.ready = Promise.resolve(this.meta);\r\n }\r\n\r\n #renderTxt(txt: string) {\r\n const lines = txt\r\n .split('\\n')\r\n .reverse()\r\n .map((t) => t.trim());\r\n\r\n const { width, height } = this.#cvs;\r\n\r\n const {\r\n color,\r\n fontSize,\r\n textBgColor,\r\n textShadow,\r\n strokeStyle,\r\n lineWidth,\r\n lineCap,\r\n lineJoin,\r\n bottomOffset,\r\n } = this.#opts;\r\n const ctx = this.#ctx;\r\n\r\n ctx.clearRect(0, 0, width, height);\r\n ctx.globalAlpha = 0.6;\r\n // 测试canvas背景\r\n // ctx.fillStyle = 'red'\r\n // ctx.fillRect(0, 0, this.#cvs.width, this.#cvs.height)\r\n\r\n let bottomDistance = bottomOffset;\r\n for (const lineStr of lines) {\r\n const txtMeas = ctx.measureText(lineStr);\r\n const centerX = width / 2;\r\n if (textBgColor != null) {\r\n ctx.shadowOffsetX = 0;\r\n ctx.shadowOffsetY = 0;\r\n ctx.shadowBlur = 0;\r\n // 字幕背景\r\n ctx.fillStyle = textBgColor;\r\n ctx.globalAlpha = 0.5;\r\n ctx.fillRect(\r\n centerX - txtMeas.actualBoundingBoxLeft - this.#linePadding,\r\n height - bottomDistance - this.#lineHeight,\r\n txtMeas.width + this.#linePadding * 2,\r\n this.#lineHeight,\r\n );\r\n } else {\r\n }\r\n\r\n ctx.shadowColor = textShadow.color;\r\n ctx.shadowOffsetX = textShadow.offsetX;\r\n ctx.shadowOffsetY = textShadow.offsetY;\r\n ctx.shadowBlur = textShadow.blur;\r\n\r\n ctx.globalAlpha = 1;\r\n\r\n if (strokeStyle != null) {\r\n ctx.lineWidth = lineWidth ?? fontSize / 6;\r\n if (lineCap != null) ctx.lineCap = lineCap;\r\n if (lineJoin != null) ctx.lineJoin = lineJoin;\r\n ctx.strokeStyle = strokeStyle;\r\n ctx.strokeText(\r\n lineStr,\r\n centerX,\r\n height - bottomDistance - this.#lineHeight + this.#linePadding,\r\n );\r\n }\r\n\r\n ctx.fillStyle = color;\r\n ctx.fillText(\r\n lineStr,\r\n centerX,\r\n height - bottomDistance - this.#lineHeight + this.#linePadding,\r\n );\r\n\r\n // 多行，底部偏移距离叠加\r\n bottomDistance += this.#lineHeight + fontSize * 0.2;\r\n }\r\n }\r\n\r\n /**\r\n * @see {@link IClip.tick}\r\n */\r\n async tick(time: number): Promise<{\r\n video?: VideoFrame;\r\n state: 'done' | 'success';\r\n }> {\r\n if (\r\n this.#lastVF != null &&\r\n time >= this.#lastVF.timestamp &&\r\n time <= this.#lastVF.timestamp + (this.#lastVF.duration ?? 0)\r\n ) {\r\n return { video: this.#lastVF.clone(), state: 'success' };\r\n }\r\n\r\n let i = 0;\r\n for (; i < this.#subtitles.length; i += 1) {\r\n if (time <= this.#subtitles[i].end) break;\r\n }\r\n\r\n const it = this.#subtitles[i] ?? this.#subtitles.at(-1);\r\n if (time > it.end) return { state: 'done' };\r\n if (time < it.start) {\r\n // 此时无字幕内容，清空画布\r\n this.#ctx.clearRect(0, 0, this.#cvs.width, this.#cvs.height);\r\n const vf = new VideoFrame(this.#cvs, {\r\n timestamp: time,\r\n // 直到下个字幕出现的时机\r\n duration: it.start - time,\r\n });\r\n this.#lastVF?.close();\r\n this.#lastVF = vf;\r\n\r\n return { video: vf.clone(), state: 'success' };\r\n }\r\n\r\n this.#renderTxt(it.text);\r\n\r\n const vf = new VideoFrame(this.#cvs, {\r\n timestamp: time,\r\n duration: it.end - time,\r\n });\r\n this.#lastVF?.close();\r\n this.#lastVF = vf;\r\n\r\n return { video: vf.clone(), state: 'success' };\r\n }\r\n\r\n /**\r\n * @see {@link IClip.split}\r\n */\r\n async split(time: number) {\r\n await this.ready;\r\n let hitIdx = -1;\r\n for (let i = 0; i < this.#subtitles.length; i++) {\r\n const sub = this.#subtitles[i];\r\n if (time > sub.start) continue;\r\n hitIdx = i;\r\n break;\r\n }\r\n if (hitIdx === -1) throw Error('Not found subtitle by time');\r\n const preSlice = this.#subtitles.slice(0, hitIdx).map((s) => ({ ...s }));\r\n let preLastIt = preSlice.at(-1);\r\n let postFirstIt = null;\r\n // 切割时间命中字幕区间，需要将当前字幕元素拆成前后两份\r\n if (preLastIt != null && preLastIt.end > time) {\r\n postFirstIt = {\r\n start: 0,\r\n end: preLastIt.end - time,\r\n text: preLastIt.text,\r\n };\r\n\r\n preLastIt.end = time;\r\n }\r\n const postSlice = this.#subtitles\r\n .slice(hitIdx)\r\n .map((s) => ({ ...s, start: s.start - time, end: s.end - time }));\r\n if (postFirstIt != null) postSlice.unshift(postFirstIt);\r\n return [\r\n new EmbedSubtitlesClip(preSlice, this.#opts),\r\n new EmbedSubtitlesClip(postSlice, this.#opts),\r\n ] as [this, this];\r\n }\r\n\r\n /**\r\n * @see {@link IClip.clone}\r\n */\r\n async clone() {\r\n return new EmbedSubtitlesClip(this.#subtitles.slice(0), this.#opts) as this;\r\n }\r\n\r\n /**\r\n * @see {@link IClip.destroy}\r\n */\r\n destroy() {\r\n this.#lastVF?.close();\r\n }\r\n}\r\n\r\n// SRT字幕格式 https://www.cnblogs.com/tocy/p/subtitle-format-srt.html\r\nfunction srtTimeToSeconds(time: string) {\r\n const match = time.match(/(\\d{2}):(\\d{2}):(\\d{2}),(\\d{3})/);\r\n if (match == null) throw Error(`time format error: ${time}`);\r\n\r\n const hours = Number(match[1]);\r\n const minutes = Number(match[2]);\r\n const seconds = Number(match[3]);\r\n const milliseconds = Number(match[4]);\r\n\r\n return hours * 60 * 60 + minutes * 60 + seconds + milliseconds / 1000;\r\n}\r\n\r\nfunction parseSrt(srt: string) {\r\n return (\r\n srt\r\n .split(/\\r|\\n/)\r\n .map((s) => s.trim())\r\n .filter((str) => str.length > 0)\r\n // 匹配时间戳标记行，匹配失败的为字幕内容\r\n .map((s) => ({\r\n lineStr: s,\r\n match: s.match(\r\n /(\\d{2}:\\d{2}:\\d{2},\\d{3}) --> (\\d{2}:\\d{2}:\\d{2},\\d{3})/,\r\n ),\r\n }))\r\n // 过滤掉时间上一行的数字标记\r\n .filter(\r\n ({ lineStr }, idx, source) =>\r\n !(/^\\d+$/.test(lineStr) && source[idx + 1]?.match != null),\r\n )\r\n // 按时间标记行聚合，拼接字幕内容到 text 字段\r\n .reduce(\r\n (acc, { lineStr, match }) => {\r\n if (match == null) {\r\n const last = acc.at(-1);\r\n if (last == null) return acc;\r\n\r\n last.text += last.text.length === 0 ? lineStr : `\\n${lineStr}`;\r\n } else {\r\n acc.push({\r\n start: srtTimeToSeconds(match[1]),\r\n end: srtTimeToSeconds(match[2]),\r\n text: '',\r\n });\r\n }\r\n\r\n return acc;\r\n },\r\n [] as Array<{\r\n start: number;\r\n end: number;\r\n text: string;\r\n }>,\r\n )\r\n );\r\n}\r\n","import mp4box, {\r\n MP4ArrayBuffer,\r\n MP4File,\r\n MP4Info,\r\n MP4Sample,\r\n} from '@webav/mp4box.js';\r\n\r\n/**\r\n * 将原始字节流转换成 MP4Sample 流\r\n */\r\nexport class SampleTransform {\r\n readable: ReadableStream<\r\n | {\r\n chunkType: 'ready';\r\n data: { info: MP4Info; file: MP4File };\r\n }\r\n | {\r\n chunkType: 'samples';\r\n data: { id: number; type: 'video' | 'audio'; samples: MP4Sample[] };\r\n }\r\n >;\r\n\r\n writable: WritableStream;\r\n\r\n #inputBufOffset = 0;\r\n\r\n constructor() {\r\n const file = mp4box.createFile();\r\n let streamCancelled = false;\r\n this.readable = new ReadableStream(\r\n {\r\n start: (ctrl) => {\r\n file.onReady = (info) => {\r\n const vTrackId = info.videoTracks[0]?.id;\r\n if (vTrackId != null)\r\n file.setExtractionOptions(vTrackId, 'video', { nbSamples: 100 });\r\n\r\n const aTrackId = info.audioTracks[0]?.id;\r\n if (aTrackId != null)\r\n file.setExtractionOptions(aTrackId, 'audio', { nbSamples: 100 });\r\n\r\n ctrl.enqueue({ chunkType: 'ready', data: { info, file } });\r\n file.start();\r\n };\r\n\r\n const releasedCnt: Record = {};\r\n file.onSamples = (id, type, samples) => {\r\n ctrl.enqueue({\r\n chunkType: 'samples',\r\n data: { id, type, samples: samples.map((s) => ({ ...s })) },\r\n });\r\n releasedCnt[id] = (releasedCnt[id] ?? 0) + samples.length;\r\n file.releaseUsedSamples(id, releasedCnt[id]);\r\n };\r\n\r\n file.onFlush = () => {\r\n ctrl.close();\r\n };\r\n },\r\n cancel: () => {\r\n file.stop();\r\n streamCancelled = true;\r\n },\r\n },\r\n {\r\n // 每条消息 100 个 samples\r\n highWaterMark: 50,\r\n },\r\n );\r\n\r\n this.writable = new WritableStream({\r\n write: async (ui8Arr) => {\r\n if (streamCancelled) {\r\n this.writable.abort();\r\n return;\r\n }\r\n\r\n const inputBuf = ui8Arr.buffer as MP4ArrayBuffer;\r\n inputBuf.fileStart = this.#inputBufOffset;\r\n this.#inputBufOffset += inputBuf.byteLength;\r\n file.appendBuffer(inputBuf);\r\n },\r\n close: () => {\r\n file.flush();\r\n file.stop();\r\n file.onFlush?.();\r\n },\r\n });\r\n }\r\n}\r\n","import mp4box, {\n MP4File,\n MP4Sample,\n SampleOpts,\n TrakBoxParser,\n} from '@webav/mp4box.js';\nimport { autoReadStream, file2stream, Log } from '@webrock/internal-utils';\nimport { tmpfile, write } from 'opfs-tools';\nimport {\n concatPCMFragments,\n extractPCM4AudioBuffer,\n extractPCM4AudioData,\n mixinPCM,\n ringSliceFloat32Array,\n} from '../av-utils';\nimport { DEFAULT_AUDIO_CONF } from '../clips';\nimport { extractFileConfig } from './mp4box-utils';\nimport { SampleTransform } from './sample-transform';\n\nfunction fixMP4BoxFileDuration(\n inMP4File: MP4File,\n): () => Promise | null> {\n let sendedBoxIdx = 0;\n const boxes = inMP4File.boxes;\n const tracks: Array<{ track: TrakBoxParser; id: number }> = [];\n let totalDuration = 0;\n\n async function write2TmpFile() {\n const buf = box2Buf(boxes, sendedBoxIdx);\n sendedBoxIdx = boxes.length;\n // 释放引用，避免内存泄露\n // todo: use unsafeReleaseMP4BoxFile\n tracks.forEach(({ track, id }) => {\n const s = track.samples.at(-1);\n if (s != null)\n totalDuration = Math.max(totalDuration, s.cts + s.duration);\n\n inMP4File.releaseUsedSamples(id, track.samples.length);\n track.samples = [];\n });\n inMP4File.mdats = [];\n inMP4File.moofs = [];\n if (buf != null) await tmpFileWriter?.write(buf);\n }\n\n let moovPrevBoxes: typeof boxes = [];\n function moovBoxReady() {\n if (moovPrevBoxes.length > 0) return true;\n\n const moovIdx = boxes.findIndex((box) => box.type === 'moov');\n if (moovIdx === -1) return false;\n\n moovPrevBoxes = boxes.slice(0, moovIdx + 1);\n sendedBoxIdx = moovIdx + 1;\n\n if (tracks.length === 0) {\n for (let i = 1; true; i += 1) {\n const track = inMP4File.getTrackById(i);\n if (track == null) break;\n tracks.push({ track, id: i });\n }\n }\n\n return true;\n }\n\n let timerId = 0;\n // 把 moov 之外的 box 先写入临时文件，待更新 duration 之后再拼接临时文件\n const postFile = tmpfile();\n let tmpFileWriter: Awaited<\n ReturnType['createWriter']>\n > | null = null;\n\n const initPromise = (async () => {\n tmpFileWriter = await postFile.createWriter();\n\n timerId = self.setInterval(() => {\n if (!moovBoxReady()) return;\n write2TmpFile();\n }, 100);\n })();\n\n let stoped = false;\n return async () => {\n if (stoped) throw Error('File exported');\n stoped = true;\n\n await initPromise;\n clearInterval(timerId);\n\n if (!moovBoxReady() || tmpFileWriter == null) return null;\n inMP4File.flush();\n await write2TmpFile();\n await tmpFileWriter?.close();\n\n const moov = moovPrevBoxes.find((box) => box.type === 'moov') as\n | typeof inMP4File.moov\n | undefined;\n if (moov == null) return null;\n\n moov.mvhd.duration = totalDuration;\n\n const rsFile = tmpfile();\n const buf = box2Buf(moovPrevBoxes, 0)!;\n await write(rsFile, buf);\n await write(rsFile, postFile, { overwrite: false });\n\n return await rsFile.stream();\n };\n\n function box2Buf(source: typeof boxes, startIdx: number): Uint8Array | null {\n if (startIdx >= source.length) return null;\n\n const ds = new mp4box.DataStream();\n ds.endianness = mp4box.DataStream.BIG_ENDIAN;\n\n for (let i = startIdx; i < source.length; i++) {\n if (source[i] === null) continue;\n source[i].write(ds);\n delete source[i];\n }\n return new Uint8Array(ds.buffer);\n }\n}\n\n/**\n * EncodedAudioChunk | EncodedVideoChunk 转换为 MP4 addSample 需要的参数\n */\nfunction chunk2MP4SampleOpts(\n chunk: EncodedAudioChunk | EncodedVideoChunk,\n): SampleOpts & {\n data: ArrayBuffer;\n} {\n const buf = new ArrayBuffer(chunk.byteLength);\n chunk.copyTo(buf);\n const dts = chunk.timestamp;\n return {\n duration: chunk.duration ?? 0,\n dts,\n cts: dts,\n is_sync: chunk.type === 'key',\n data: buf,\n };\n}\n\n/**\n * 快速拼接多个mp4 文件流，要求所有 mp4 的属性一致，\n * 属性包括（不限于）：音视频编码格式、分辨率、采样率\n *\n * @param streams 一个包含 Uint8Array 的可读流数组。\n * @returns 返回一个 Promise，该 Promise 在解析时返回一个包含合并后的 MP4 数据的可读流。\n * @throws 如果无法从流生成文件，将抛出错误。\n *\n * @example\n * const streams = [stream1, stream2, stream3];\n * const resultStream = await fastConcatMP4(streams);\n */\nexport async function fastConcatMP4(\n streams: ReadableStream[],\n): Promise> {\n const outfile = mp4box.createFile();\n\n const dumpFile = fixMP4BoxFileDuration(outfile);\n await concatStreamsToMP4BoxFile(streams, outfile);\n const outStream = await dumpFile();\n if (outStream == null) throw Error('Can not generate file from streams');\n return outStream;\n}\n\nasync function concatStreamsToMP4BoxFile(\n streams: ReadableStream[],\n outfile: MP4File,\n) {\n let vTrackId = 0;\n let vDTS = 0;\n let vCTS = 0;\n let aTrackId = 0;\n let aDTS = 0;\n let aCTS = 0;\n // ts bug, 不能正确识别类型\n let lastVSamp: any = null;\n let lastASamp: any = null;\n for (const stream of streams) {\n await new Promise(async (resolve) => {\n autoReadStream(stream.pipeThrough(new SampleTransform()), {\n onDone: resolve,\n onChunk: async ({ chunkType, data }) => {\n if (chunkType === 'ready') {\n const { videoTrackConf, audioTrackConf } = extractFileConfig(\n data.file,\n data.info,\n );\n if (vTrackId === 0 && videoTrackConf != null) {\n vTrackId = outfile.addTrack(videoTrackConf);\n }\n if (aTrackId === 0 && audioTrackConf != null) {\n aTrackId = outfile.addTrack(audioTrackConf);\n }\n } else if (chunkType === 'samples') {\n const { type, samples } = data;\n const trackId = type === 'video' ? vTrackId : aTrackId;\n const offsetDTS = type === 'video' ? vDTS : aDTS;\n const offsetCTS = type === 'video' ? vCTS : aCTS;\n\n samples.forEach((s) => {\n const ab = s.data.buffer.slice(\n s.data.byteOffset,\n s.data.byteOffset + s.data.byteLength,\n );\n outfile.addSample(\n trackId,\n ab instanceof ArrayBuffer ? ab : new ArrayBuffer(0),\n {\n duration: s.duration,\n dts: s.dts + offsetDTS,\n cts: s.cts + offsetCTS,\n is_sync: s.is_sync,\n },\n );\n });\n\n const lastSamp = samples.at(-1);\n if (lastSamp == null) return;\n if (type === 'video') {\n lastVSamp = lastSamp;\n } else if (type === 'audio') {\n lastASamp = lastSamp;\n }\n }\n },\n });\n });\n if (lastVSamp != null) {\n vDTS += lastVSamp.dts;\n vCTS += lastVSamp.cts;\n }\n if (lastASamp != null) {\n aDTS += lastASamp.dts;\n aCTS += lastASamp.cts;\n }\n }\n}\n\n/**\n * 为 WebAV 生成的 fmp4 文件设置正确的时长值\n */\nexport async function fixFMP4Duration(\n stream: ReadableStream,\n): Promise> {\n return await fastConcatMP4([stream]);\n}\n\n/**\n * 创建 MP4 音频样本解码器。\n * @param adConf - 音频解码器配置参数 {@link AudioDecoderConfig}。\n * @returns 返回一个对象，包含 `decode` 和 `close` 方法。\n * - `decode` 方法用于解码 MP4 音频样本，返回解码后的音频数据数组。\n * - `close` 方法用于关闭音频解码器。\n */\nfunction createMP4AudioSampleDecoder(\n adConf: Parameters[0],\n) {\n let cacheAD: AudioData[] = [];\n const adDecoder = new AudioDecoder({\n output: (ad) => {\n cacheAD.push(ad);\n },\n error: Log.error,\n });\n adDecoder.configure(adConf);\n\n return {\n decode: async (ss: MP4Sample[]) => {\n ss.forEach((s) => {\n adDecoder.decode(\n new EncodedAudioChunk({\n type: s.is_sync ? 'key' : 'delta',\n timestamp: (1e6 * s.cts) / s.timescale,\n duration: (1e6 * s.duration) / s.timescale,\n data: s.data,\n }),\n );\n });\n\n await adDecoder.flush();\n\n const rs = cacheAD;\n cacheAD = [];\n\n return rs;\n },\n close: () => {\n adDecoder.close();\n },\n };\n}\n\n// 音频编码与解码API有很大区别，\n// 是因为编码中途调用 AudioEncoder.flush ，会导致声音听起来卡顿\nfunction createMP4AudioSampleEncoder(\n aeConf: Parameters[0],\n onOutput: (s: ReturnType) => void,\n) {\n const encoderConf = {\n codec: aeConf.codec,\n sampleRate: aeConf.sampleRate,\n numberOfChannels: aeConf.numberOfChannels,\n } as const;\n\n const adEncoder = new AudioEncoder({\n output: (chunk) => {\n onOutput(chunk2MP4SampleOpts(chunk));\n },\n error: (err) => {\n Log.error('AudioEncoder error:', err, ', config:', encoderConf);\n },\n });\n\n adEncoder.configure(encoderConf);\n\n // 保留一个音频数据，用于最后做声音淡出\n let lastData: { data: Float32Array; ts: number } | null = null;\n\n function createAD(data: Float32Array, ts: number) {\n return new AudioData({\n timestamp: ts,\n numberOfChannels: aeConf.numberOfChannels,\n numberOfFrames: data.length / aeConf.numberOfChannels,\n sampleRate: aeConf.sampleRate,\n format: 'f32-planar',\n data,\n });\n }\n return {\n encode: async (data: Float32Array, ts: number) => {\n if (lastData != null) {\n adEncoder.encode(createAD(lastData.data, lastData.ts));\n }\n lastData = { data, ts };\n },\n stop: async () => {\n if (lastData != null) {\n // 副作用修改数据\n audioFade(lastData.data, aeConf.numberOfChannels, aeConf.sampleRate);\n adEncoder.encode(createAD(lastData.data, lastData.ts));\n lastData = null;\n }\n await adEncoder.flush();\n adEncoder.close();\n },\n };\n}\n\n/**\n * 音频线性淡出，避免 POP 声\n * 副作用调整音量值\n */\nfunction audioFade(pcmData: Float32Array, chanCnt: number, sampleRate: number) {\n const dataLen = pcmData.length - 1;\n // 避免超出边界，最长 500ms 的淡出时间\n const fadeLen = Math.min(sampleRate / 2, dataLen);\n for (let i = 0; i < fadeLen; i++) {\n for (let j = 1; j <= chanCnt; j++) {\n // 从尾部开始，调整每个声道音量值\n pcmData[Math.floor(dataLen / j) - i] *= i / fadeLen;\n }\n }\n}\n\n/**\n * 视频配音；混合 MP4 与音频文件，仅重编码音频，视频轨道不变\n * @param mp4Stream - MP4 流\n * @param audio - 音频信息\n * @param audio.stream - 音频数据流\n * @param audio.volume - 音频音量\n * @param audio.loop - 音频时长小于视频时，是否循环使用音频流\n * @returns 输出混合后的音频流\n */\nexport function mixinMP4AndAudio(\n mp4Stream: ReadableStream,\n audio: {\n stream: ReadableStream;\n volume: number;\n loop: boolean;\n },\n) {\n Log.info('mixinMP4AndAudio, opts:', {\n volume: audio.volume,\n loop: audio.loop,\n });\n\n const outfile = mp4box.createFile();\n const { stream: outStream, stop: stopOut } = file2stream(outfile, 500);\n\n let audioSampleDecoder: ReturnType<\n typeof createMP4AudioSampleDecoder\n > | null = null;\n\n let audioSampleEncoder: ReturnType<\n typeof createMP4AudioSampleEncoder\n > | null = null;\n\n let inputAudioPCM: Float32Array[] = [];\n\n let vTrackId = 0;\n let aTrackId = 0;\n let audioOffset = 0;\n let mp4HasAudio = true;\n let sampleRate = DEFAULT_AUDIO_CONF.sampleRate as number;\n autoReadStream(mp4Stream.pipeThrough(new SampleTransform()), {\n onDone: async () => {\n await audioSampleEncoder?.stop();\n audioSampleDecoder?.close();\n stopOut();\n },\n onChunk: async ({ chunkType, data }) => {\n if (chunkType === 'ready') {\n const { videoTrackConf, audioTrackConf, audioDecoderConf } =\n extractFileConfig(data.file, data.info);\n if (vTrackId === 0 && videoTrackConf != null) {\n vTrackId = outfile.addTrack(videoTrackConf);\n }\n\n const safeAudioTrackConf = audioTrackConf ?? {\n timescale: 1e6,\n samplerate: sampleRate,\n channel_count: DEFAULT_AUDIO_CONF.channelCount,\n hdlr: 'soun',\n name: 'SoundHandler',\n type: 'mp4a',\n };\n if (aTrackId === 0) {\n aTrackId = outfile.addTrack(safeAudioTrackConf);\n sampleRate = audioTrackConf?.samplerate ?? sampleRate;\n mp4HasAudio = audioTrackConf == null ? false : true;\n }\n const audioCtx = new AudioContext({ sampleRate });\n inputAudioPCM = extractPCM4AudioBuffer(\n await audioCtx.decodeAudioData(\n await new Response(audio.stream).arrayBuffer(),\n ),\n );\n\n if (audioDecoderConf != null) {\n audioSampleDecoder = createMP4AudioSampleDecoder(audioDecoderConf);\n }\n audioSampleEncoder = createMP4AudioSampleEncoder(\n audioDecoderConf ?? {\n codec:\n safeAudioTrackConf.type === 'mp4a'\n ? DEFAULT_AUDIO_CONF.codec\n : safeAudioTrackConf.type,\n numberOfChannels: safeAudioTrackConf.channel_count,\n sampleRate: safeAudioTrackConf.samplerate,\n },\n (s) => outfile.addSample(aTrackId, s.data, s),\n );\n } else if (chunkType === 'samples') {\n const { id, type, samples } = data;\n if (type === 'video') {\n samples.forEach((s) => {\n const ab = s.data.buffer.slice(\n s.data.byteOffset,\n s.data.byteOffset + s.data.byteLength,\n );\n outfile.addSample(\n id,\n ab instanceof ArrayBuffer ? ab : new ArrayBuffer(0),\n s,\n );\n });\n\n if (!mp4HasAudio) await addInputAudio2Track(samples);\n return;\n }\n\n if (type === 'audio') await mixinAudioSampleAndInputPCM(samples);\n }\n },\n });\n\n function getInputAudioSlice(len: number) {\n const rs = inputAudioPCM.map((chanBuf) =>\n audio.loop\n ? ringSliceFloat32Array(chanBuf, audioOffset, audioOffset + len)\n : chanBuf.slice(audioOffset, audioOffset + len),\n );\n audioOffset += len;\n\n if (audio.volume !== 1) {\n for (const buf of rs)\n for (let i = 0; i < buf.length; i++) buf[i] *= audio.volume;\n }\n\n return rs;\n }\n\n async function addInputAudio2Track(vdieoSamples: MP4Sample[]) {\n const firstSamp = vdieoSamples[0];\n const lastSamp = vdieoSamples[vdieoSamples.length - 1];\n const pcmLength = Math.floor(\n ((lastSamp.cts + lastSamp.duration - firstSamp.cts) /\n lastSamp.timescale) *\n sampleRate,\n );\n const audioDataBuf = mixinPCM([getInputAudioSlice(pcmLength)]);\n if (audioDataBuf.length === 0) return;\n audioSampleEncoder?.encode(\n audioDataBuf,\n (firstSamp.cts / firstSamp.timescale) * 1e6,\n );\n }\n\n async function mixinAudioSampleAndInputPCM(samples: MP4Sample[]) {\n if (audioSampleDecoder == null) return;\n\n // 1. 先解码mp4音频\n // [[chan0, chan1], [chan0, chan1]...]\n const pcmFragments = (await audioSampleDecoder.decode(samples)).map(\n extractPCM4AudioData,\n );\n // [chan0, chan1]\n const mp4AudioPCM = concatPCMFragments(pcmFragments);\n const inputAudioPCM = getInputAudioSlice(mp4AudioPCM[0].length);\n const firstSamp = samples[0];\n\n // 3. 重编码音频\n audioSampleEncoder?.encode(\n // 2. 混合输入的音频\n mixinPCM([mp4AudioPCM, inputAudioPCM]),\n (firstSamp.cts / firstSamp.timescale) * 1e6,\n );\n }\n\n return outStream;\n}\n","import {\n EventTool,\n file2stream,\n Log,\n recodemux,\n} from '@webrock/internal-utils';\nimport { sleep } from './av-utils';\nimport { DEFAULT_AUDIO_CONF } from './clips';\nimport { OffscreenSprite } from './sprite/offscreen-sprite';\n\nexport interface ICombinatorOpts {\n width?: number;\n height?: number;\n bitrate?: number;\n fps?: number;\n bgColor?: string;\n videoCodec?: string;\n /**\n * false 合成的视频文件中排除音轨\n */\n audio?: false;\n /**\n * 向输出的视频中写入 meta tags 数据\n */\n metaDataTags?: Record;\n /**\n * 不安全，随时可能废弃\n */\n __unsafe_hardwareAcceleration__?: HardwarePreference;\n}\n\nlet COM_ID = 0;\n\n/**\n * 避免 VideoEncoder 队列中的 VideoFrame 过多，打爆显存\n */\nasync function letEncoderCalmDown(getQSize: () => number) {\n if (getQSize() > 50) {\n await sleep(15);\n await letEncoderCalmDown(getQSize);\n }\n}\n\n/**\n * 视频合成器；能添加多个 {@link OffscreenSprite}，根据它们位置、层级、时间偏移等信息，合成输出为视频文件\n * @see [视频合成](https://webav-tech.github.io/WebAV/demo/2_1-concat-video)\n * @see [视频配音](https://webav-tech.github.io/WebAV/demo/2_2-video-add-audio)\n * @example\n * const spr1 = new OffscreenSprite(\n * new MP4Clip((await fetch('')).body),\n * );\n * const spr2 = new OffscreenSprite(\n * new AudioClip((await fetch('