{"version":3,"file":"av-cliper.umd.cjs","sources":["../src/dom-utils.ts","../src/av-utils.ts","../src/clips/iclip.ts","../src/mp4-utils/mp4box-utils.ts","../src/clips/mp4-clip.ts","../src/clips/img-clip.ts","../src/clips/audio-clip.ts","../src/clips/media-stream-clip.ts","../src/clips/embed-subtitles-clip.ts","../src/mp4-utils/sample-transform.ts","../src/mp4-utils/index.ts","../src/chromakey.ts","../src/sprite/rect.ts","../src/sprite/base-sprite.ts","../src/sprite/offscreen-sprite.ts","../src/sprite/visible-sprite.ts","../src/combinator.ts"],"sourcesContent":["// 在主线程中执行的 工具函数\n\n/**\n * 创建一个新的 HTML 元素\n * @param tagName - 要创建的元素的标签名\n * @returns 新创建的 HTML 元素\n */\nexport function createEl(tagName: string): HTMLElement {\n return document.createElement(tagName);\n}\n\nfunction arrayBufferToBase64(buffer: ArrayBuffer) {\n var binary = '';\n var bytes = new Uint8Array(buffer);\n var len = bytes.byteLength;\n for (let i = 0; i < len; i++) {\n binary += String.fromCharCode(bytes[i]);\n }\n return window.btoa(binary);\n}\n\n/**\n * 将文本渲染为图片\n * @param txt - 要渲染的文本\n * @param cssText - 应用于文本的 CSS 样式\n * @returns 渲染后的图片元素\n */\nexport async function renderTxt2Img(\n txt: string,\n cssText: string,\n opts: {\n font?: { name: string; url: string };\n onCreated?: (el: HTMLElement) => void;\n } = {},\n): Promise {\n const div = createEl('pre');\n div.style.cssText = `margin: 0; ${cssText}; position: fixed;`;\n div.textContent = txt;\n document.body.appendChild(div);\n opts.onCreated?.(div);\n\n const { width, height } = div.getBoundingClientRect();\n // 计算出 rect，立即从dom移除\n div.remove();\n\n const img = new Image();\n img.width = width;\n img.height = height;\n const fontFaceStr =\n opts.font == null\n ? ''\n : `\n @font-face {\n font-family: '${opts.font.name}';\n src: url('data:font/woff2;base64,${arrayBufferToBase64(await (await fetch(opts.font.url)).arrayBuffer())}') format('woff2');\n }\n `;\n const svgStr = `\n \n \n \n
${div.outerHTML}
\n \n \n `\n .replace(/\\t/g, '')\n .replace(/#/g, '%23');\n\n img.src = `data:image/svg+xml;charset=utf-8,${svgStr}`;\n\n await new Promise((resolve) => {\n img.onload = resolve;\n });\n return img;\n}\n\n/**\n * 将文本渲染为 {@link ImageBitmap}，用来创建 {@link ImgClip}\n * @param txt - 要渲染的文本\n * @param cssText - 应用于文本的 CSS 样式\n * @param opts - 选项\n * @param opts.font - 自定义字体\n * @param opts.onCreated - 创建完成后的回调\n *\n * @example\n * new ImgClip(\n * await renderTxt2ImgBitmap(\n * '水印',\n * `font-size:40px; color: white; text-shadow: 2px 2px 6px red; font-family: CustomFont;`,\n * {\n * font: {\n * name: 'CustomFont',\n * url: '/CustomFont.ttf',\n * },\n * },\n * )\n * )\n */\nexport async function renderTxt2ImgBitmap(\n txt: string,\n cssText: string,\n opts: {\n font?: { name: string; url: string };\n onCreated?: (el: HTMLElement) => void;\n } = {},\n): Promise {\n const imgEl = await renderTxt2Img(txt, cssText, opts);\n const cvs = new OffscreenCanvas(imgEl.width, imgEl.height);\n const ctx = cvs.getContext('2d');\n ctx?.drawImage(imgEl, 0, 0, imgEl.width, imgEl.height);\n return await createImageBitmap(cvs);\n}\n","// 能同时在 worker 和主线程中运行的工具函数\n\nimport { workerTimer } from '@webav/internal-utils';\nimport * as waveResampler from 'wave-resampler';\n\nimport { Log } from '@webav/internal-utils';\n\nif (import.meta.env?.DEV) {\n Log.setLogLevel(Log.debug);\n}\n\nif (import.meta.env?.MODE === 'test') {\n Log.setLogLevel(Log.warn);\n}\n\n/**\n * 合并（串联）多个 Float32Array，通常用于合并 PCM 数据\n */\nexport function concatFloat32Array(bufs: Float32Array[]): Float32Array {\n const rs = new Float32Array(\n bufs.map((buf) => buf.length).reduce((a, b) => a + b),\n );\n\n let offset = 0;\n for (const buf of bufs) {\n rs.set(buf, offset);\n offset += buf.length;\n }\n\n return rs;\n}\n\n/**\n * 将小片段的 PCM 合并成一个大片段\n * @param fragments 小片段 PCM，子元素是不同声道的原始 PCM 数据\n */\nexport function concatPCMFragments(\n fragments: Float32Array[][],\n): Float32Array[] {\n // fragments: [[chan0, chan1], [chan0, chan1]...]\n // chanListPCM: [[chan0, chan0...], [chan1, chan1...]]\n const chanListPCM: Float32Array[][] = [];\n for (let i = 0; i < fragments.length; i += 1) {\n for (let j = 0; j < fragments[i].length; j += 1) {\n if (chanListPCM[j] == null) chanListPCM[j] = [];\n chanListPCM[j].push(fragments[i][j]);\n }\n }\n // [bigChan0, bigChan1]\n return chanListPCM.map(concatFloat32Array);\n}\n\n/**\n * 从 AudioData 中提取 PCM 数据的工具函数\n */\nexport function extractPCM4AudioData(ad: AudioData): Float32Array[] {\n if (ad.format === 'f32-planar') {\n const rs = [];\n for (let idx = 0; idx < ad.numberOfChannels; idx += 1) {\n const chanBufSize = ad.allocationSize({ planeIndex: idx });\n const chanBuf = new ArrayBuffer(chanBufSize);\n ad.copyTo(chanBuf, { planeIndex: idx });\n rs.push(new Float32Array(chanBuf));\n }\n return rs;\n } else if (ad.format === 'f32') {\n const buf = new ArrayBuffer(ad.allocationSize({ planeIndex: 0 }));\n ad.copyTo(buf, { planeIndex: 0 });\n return convertF32ToPlanar(new Float32Array(buf), ad.numberOfChannels);\n } else if (ad.format === 's16') {\n const buf = new ArrayBuffer(ad.allocationSize({ planeIndex: 0 }));\n ad.copyTo(buf, { planeIndex: 0 });\n return convertS16ToF32Planar(new Int16Array(buf), ad.numberOfChannels);\n }\n throw Error('Unsupported audio data format');\n}\n\n/**\n * Convert s16 PCM to f32-planar\n * @param pcmS16Data - The s16 PCM data.\n * @param numChannels - Number of audio channels.\n * @returns An array of Float32Array, each containing the audio data for one channel.\n */\nfunction convertS16ToF32Planar(pcmS16Data: Int16Array, numChannels: number) {\n const numSamples = pcmS16Data.length / numChannels;\n const planarData = Array.from(\n { length: numChannels },\n () => new Float32Array(numSamples),\n );\n\n for (let i = 0; i < numSamples; i++) {\n for (let channel = 0; channel < numChannels; channel++) {\n const sample = pcmS16Data[i * numChannels + channel];\n planarData[channel][i] = sample / 32768; // Normalize to range [-1.0, 1.0]\n }\n }\n\n return planarData;\n}\n\nfunction convertF32ToPlanar(pcmF32Data: Float32Array, numChannels: number) {\n const numSamples = pcmF32Data.length / numChannels;\n const planarData = Array.from(\n { length: numChannels },\n () => new Float32Array(numSamples),\n );\n\n for (let i = 0; i < numSamples; i++) {\n for (let channel = 0; channel < numChannels; channel++) {\n planarData[channel][i] = pcmF32Data[i * numChannels + channel];\n }\n }\n\n return planarData;\n}\n\n/**\n * 从 AudioBuffer 中提取 PCM\n */\nexport function extractPCM4AudioBuffer(ab: AudioBuffer): Float32Array[] {\n return Array(ab.numberOfChannels)\n .fill(0)\n .map((_, idx) => {\n return ab.getChannelData(idx);\n });\n}\n\n/**\n * 调整音频数据的音量\n * @param ad - 要调整的音频对象\n * @param volume - 音量调整系数（0.0 - 1.0）\n * @returns 调整音量后的新音频数据\n */\nexport function adjustAudioDataVolume(ad: AudioData, volume: number) {\n const data = new Float32Array(\n concatFloat32Array(extractPCM4AudioData(ad)),\n ).map((v) => v * volume);\n const newAd = new AudioData({\n sampleRate: ad.sampleRate,\n numberOfChannels: ad.numberOfChannels,\n timestamp: ad.timestamp,\n format: ad.format!,\n numberOfFrames: ad.numberOfFrames,\n data,\n });\n ad.close();\n return newAd;\n}\n\n/**\n * 解码图像流，返回一个视频帧数组。\n *\n * @param stream - 包含图像数据的可读流。\n * @param type - 图像的 MIME 类型，例如 'image/jpeg'。\n *\n * @returns 返回一个 Promise，该 Promise 在解码完成后解析为 {@link VideoFrame} 数组。\n *\n * @see [解码动图](https://webav-tech.github.io/WebAV/demo/1_3-decode-image)\n *\n * @example\n *\n * const frames = await decodeImg(\n * (await fetch('')).body!,\n * `image/gif`,\n * );\n */\nexport async function decodeImg(\n stream: ReadableStream,\n type: string,\n): Promise {\n const init = {\n type,\n data: stream,\n };\n const imageDecoder = new ImageDecoder(init);\n\n await Promise.all([imageDecoder.completed, imageDecoder.tracks.ready]);\n\n let frameCnt = imageDecoder.tracks.selectedTrack?.frameCount ?? 1;\n\n const rs: VideoFrame[] = [];\n for (let i = 0; i < frameCnt; i += 1) {\n rs.push((await imageDecoder.decode({ frameIndex: i })).image);\n }\n return rs;\n}\n\n/**\n * 混合双通道音轨的 PCM 数据，并将多声道并排成一个 Float32Array 输出\n * @param audios - 一个二维数组，每个元素是一个 Float32Array 数组，代表一个音频流的 PCM 数据。\n * 每个 Float32Array 数组的第一个元素是左声道数据，第二个元素（如果有）是右声道数据。\n * 如果只有左声道数据，则右声道将复用左声道数据。\n *\n * @returns 返回一个 Float32Array，返回结果是将这个一个音轨的左右声道并排成 Float32Array。\n *\n * @example\n *\n * const audios = [\n * [new Float32Array([0.1, 0.2, 0.3]), new Float32Array([0.4, 0.5, 0.6])],\n * [new Float32Array([0.7, 0.8, 0.9])],\n * ];\n * const mixed = mixinPCM(audios);\n */\nexport function mixinPCM(audios: Float32Array[][]): Float32Array {\n const maxLen = Math.max(...audios.map((a) => a[0]?.length ?? 0));\n const data = new Float32Array(maxLen * 2);\n\n for (let bufIdx = 0; bufIdx < maxLen; bufIdx++) {\n let chan0 = 0;\n let chan1 = 0;\n for (let trackIdx = 0; trackIdx < audios.length; trackIdx++) {\n const _c0 = audios[trackIdx][0]?.[bufIdx] ?? 0;\n // 如果是单声道 PCM，第二声道复用第一声道数据\n const _c1 = audios[trackIdx][1]?.[bufIdx] ?? _c0;\n chan0 += _c0;\n chan1 += _c1;\n }\n data[bufIdx] = chan0;\n data[bufIdx + maxLen] = chan1;\n }\n\n return data;\n}\n\n/**\n * 对 PCM 音频数据进行重采样。\n *\n * @param pcmData - 一个 Float32Array 数组，每个元素代表一个声道的 PCM 数据。\n * @param curRate - 当前的采样率。\n * @param target - 目标参数对象。\n * @param target.rate - 目标采样率。\n * @param target.chanCount - 目标声道数。\n *\n * @returns 返回一个 Promise，该 Promise 在重采样完成后解析为一个 Float32Array 数组，每个元素代表一个声道的 PCM 数据。\n *\n * @example\n *\n * const pcmData = [new Float32Array([0.1, 0.2, 0.3]), new Float32Array([0.4, 0.5, 0.6])];\n * const curRate = 44100;\n * const target = { rate: 48000, chanCount: 2 };\n * const resampled = await audioResample(pcmData, curRate, target);\n */\nexport async function audioResample(\n pcmData: Float32Array[],\n curRate: number,\n target: {\n rate: number;\n chanCount: number;\n },\n): Promise {\n const chanCnt = pcmData.length;\n const emptyPCM = Array(target.chanCount)\n .fill(0)\n .map(() => new Float32Array(0));\n if (chanCnt === 0) return emptyPCM;\n\n const len = Math.max(...pcmData.map((c) => c.length));\n if (len === 0) return emptyPCM;\n\n // The Worker scope does not have access to OfflineAudioContext\n if (globalThis.OfflineAudioContext == null) {\n return pcmData.map(\n (p) =>\n new Float32Array(\n waveResampler.resample(p, curRate, target.rate, {\n method: 'sinc',\n LPF: false,\n }),\n ),\n );\n }\n\n const ctx = new globalThis.OfflineAudioContext(\n target.chanCount,\n (len * target.rate) / curRate,\n target.rate,\n );\n const abSource = ctx.createBufferSource();\n const ab = ctx.createBuffer(chanCnt, len, curRate);\n pcmData.forEach((d, idx) => ab.copyToChannel(d, idx));\n\n abSource.buffer = ab;\n abSource.connect(ctx.destination);\n abSource.start();\n\n return extractPCM4AudioBuffer(await ctx.startRendering());\n}\n\n/**\n * 使当前执行环境暂停一段时间。\n * @param time - 暂停的时间，单位为毫秒。\n * @example\n * await sleep(1000); // 暂停 1 秒\n */\nexport function sleep(time: number): Promise {\n return new Promise((resolve) => {\n const stop = workerTimer(() => {\n stop();\n resolve();\n }, time);\n });\n}\n\n/**\n * 从给定的 Float32Array 中提取一个环形切片，超出边界从 0 开始循环\n *\n * 主要用于截取 PCM 实现循环播放\n *\n * @param data - 输入的 Float32Array。\n * @param start - 切片的开始索引。\n * @param end - 切片的结束索引。\n * @returns - 返回一个新的 Float32Array，包含从 start 到 end 的数据。\n *\n * @example\n * const data = new Float32Array([0, 1, 2, 3, 4, 5]);\n * ringSliceFloat32Array(data, 4, 6); // => Float32Array [4, 5, 0]\n */\nexport function ringSliceFloat32Array(\n data: Float32Array,\n start: number,\n end: number,\n): Float32Array {\n const cnt = end - start;\n const rs = new Float32Array(cnt);\n let i = 0;\n while (i < cnt) {\n rs[i] = data[(start + i) % data.length];\n i += 1;\n }\n return rs;\n}\n\n/**\n * 改变 PCM 数据的播放速率，1 表示正常播放，0.5 表示播放速率减半，2 表示播放速率加倍\n */\nexport function changePCMPlaybackRate(\n pcmData: Float32Array,\n playbackRate: number,\n) {\n // 计算新的采样率\n const newLength = Math.floor(pcmData.length / playbackRate);\n const newPcmData = new Float32Array(newLength);\n\n // 线性插值\n for (let i = 0; i < newLength; i++) {\n // 原始数据中的位置\n const originalIndex = i * playbackRate;\n const intIndex = Math.floor(originalIndex);\n const frac = originalIndex - intIndex;\n\n // 边界检查\n if (intIndex + 1 < pcmData.length) {\n newPcmData[i] =\n pcmData[intIndex] * (1 - frac) + pcmData[intIndex + 1] * frac;\n } else {\n newPcmData[i] = pcmData[intIndex]; // 最后一个样本\n }\n }\n\n return newPcmData;\n}\n","interface IClipMeta {\n width: number;\n height: number;\n duration: number;\n}\n\n/**\n * 所有素材需要实现的接口\n *\n * 素材（Clip）是不同数据类型的抽象，给其他模块提供数据\n *\n * WebAV 内置了 {@link MP4Clip}, {@link AudioClip}, {@link ImgClip}, {@link MediaStreamClip} 等常用素材，用于给 {@link Combinator} {@link AVCanvas} 提供数据\n *\n * 你只需实现该接口即可自定义素材，拥有最大的灵活度来生成视频内容，比如动画、转场效果等\n * @see [自定义素材](https://webav-tech.github.io/WebAV/demo/2_6-custom-clip)\n *\n */\nexport interface IClip {\n /**\n * 从素材中提取指定时间数据\n * @param time 时间，单位 微秒\n */\n tick: (time: number) => Promise<{\n video?: VideoFrame | ImageBitmap | null;\n audio?: Float32Array[];\n state: 'done' | 'success';\n }>;\n\n /**\n * 当素材准备完成，ready 会切换到 resolved 状态\n */\n readonly ready: Promise;\n\n /**\n * 数据元数据\n */\n readonly meta: IClipMeta;\n\n /**\n * clone，返回一个新素材\n */\n clone: () => Promise;\n\n /**\n * 按指定时间切割，返回该时刻前后两个新素材，常用于剪辑场景按时间分割素材\n *\n * 该方法不会破坏原素材的数据\n *\n * @param time 时间，微秒\n * @returns\n */\n split?: (time: number) => Promise<[this, this]>;\n\n /**\n * 销毁实例，释放资源\n */\n destroy: () => void;\n}\n\n/**\n * 默认的音频设置，⚠️ 不要变更它的值 ⚠️\n */\nexport const DEFAULT_AUDIO_CONF = {\n sampleRate: 48000,\n channelCount: 2,\n codec: 'mp4a.40.2',\n} as const;\n","import mp4box, {\n AudioTrackOpts,\n ESDSBoxParser,\n MP4ABoxParser,\n MP4ArrayBuffer,\n MP4File,\n MP4Info,\n MP4Sample,\n TrakBoxParser,\n VideoTrackOpts,\n} from '@webav/mp4box.js';\nimport { file } from 'opfs-tools';\nimport { DEFAULT_AUDIO_CONF } from '../clips';\n\nexport function extractFileConfig(file: MP4File, info: MP4Info) {\n const vTrack = info.videoTracks[0];\n const rs: {\n videoTrackConf?: VideoTrackOpts;\n videoDecoderConf?: Parameters[0];\n audioTrackConf?: AudioTrackOpts;\n audioDecoderConf?: Parameters[0];\n } = {};\n if (vTrack != null) {\n const videoDesc = parseVideoCodecDesc(file.getTrackById(vTrack.id))?.buffer;\n const { descKey, type } = vTrack.codec.startsWith('avc1')\n ? { descKey: 'avcDecoderConfigRecord', type: 'avc1' }\n : vTrack.codec.startsWith('hvc1')\n ? { descKey: 'hevcDecoderConfigRecord', type: 'hvc1' }\n : { descKey: '', type: '' };\n if (descKey !== '') {\n rs.videoTrackConf = {\n timescale: vTrack.timescale,\n duration: vTrack.duration,\n width: vTrack.video.width,\n height: vTrack.video.height,\n brands: info.brands,\n type,\n [descKey]: videoDesc,\n };\n }\n\n rs.videoDecoderConf = {\n codec: vTrack.codec,\n codedHeight: vTrack.video.height,\n codedWidth: vTrack.video.width,\n description: videoDesc as ArrayBuffer,\n };\n }\n\n const aTrack = info.audioTracks[0];\n if (aTrack != null) {\n const esdsBox = getESDSBoxFromMP4File(file);\n rs.audioTrackConf = {\n timescale: aTrack.timescale,\n samplerate: aTrack.audio.sample_rate,\n channel_count: aTrack.audio.channel_count,\n hdlr: 'soun',\n type: aTrack.codec.startsWith('mp4a') ? 'mp4a' : aTrack.codec,\n description: getESDSBoxFromMP4File(file),\n };\n rs.audioDecoderConf = {\n codec: aTrack.codec.startsWith('mp4a')\n ? DEFAULT_AUDIO_CONF.codec\n : aTrack.codec,\n numberOfChannels: aTrack.audio.channel_count,\n sampleRate: aTrack.audio.sample_rate,\n ...(esdsBox == null ? {} : parseAudioInfo4ESDSBox(esdsBox)),\n };\n }\n return rs;\n}\n\n// track is H.264, H.265 or VPX.\nfunction parseVideoCodecDesc(track: TrakBoxParser): Uint8Array | undefined {\n for (const entry of track.mdia.minf.stbl.stsd.entries) {\n // @ts-expect-error\n const box = entry.avcC ?? entry.hvcC ?? entry.av1C ?? entry.vpcC;\n if (box != null) {\n const stream = new mp4box.DataStream(\n undefined,\n 0,\n mp4box.DataStream.BIG_ENDIAN,\n );\n box.write(stream);\n return new Uint8Array(stream.buffer.slice(8)); // Remove the box header.\n }\n }\n return undefined;\n}\n\nfunction getESDSBoxFromMP4File(file: MP4File, codec = 'mp4a') {\n const mp4aBox = file.moov?.traks\n .map((t) => t.mdia.minf.stbl.stsd.entries)\n .flat()\n .find(({ type }) => type === codec) as MP4ABoxParser;\n\n return mp4aBox?.esds;\n}\n\n// 解决封装层音频信息标识错误，导致解码异常\nfunction parseAudioInfo4ESDSBox(esds: ESDSBoxParser) {\n const decoderConf = esds.esd.descs[0]?.descs[0];\n if (decoderConf == null) return {};\n\n const [byte1, byte2] = decoderConf.data;\n // sampleRate 是第一字节后 3bit + 第二字节前 1bit\n const sampleRateIdx = ((byte1 & 0x07) << 1) + (byte2 >> 7);\n // numberOfChannels 是第二字节 [2, 5] 4bit\n const numberOfChannels = (byte2 & 0x7f) >> 3;\n const sampleRateEnum = [\n 96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025,\n 8000, 7350,\n ] as const;\n return {\n sampleRate: sampleRateEnum[sampleRateIdx],\n numberOfChannels,\n };\n}\n\n/**\n * 快速解析 mp4 文件，如果是非 fMP4 格式，会优先解析 moov box（略过 mdat）避免占用过多内存\n */\nexport async function quickParseMP4File(\n reader: Awaited['createReader']>>,\n onReady: (data: { mp4boxFile: MP4File; info: MP4Info }) => void,\n onSamples: (\n id: number,\n sampleType: 'video' | 'audio',\n samples: MP4Sample[],\n ) => void,\n) {\n const mp4boxFile = mp4box.createFile(false);\n mp4boxFile.onReady = (info) => {\n onReady({ mp4boxFile, info });\n const vTrackId = info.videoTracks[0]?.id;\n if (vTrackId != null)\n mp4boxFile.setExtractionOptions(vTrackId, 'video', { nbSamples: 100 });\n\n const aTrackId = info.audioTracks[0]?.id;\n if (aTrackId != null)\n mp4boxFile.setExtractionOptions(aTrackId, 'audio', { nbSamples: 100 });\n\n mp4boxFile.start();\n };\n mp4boxFile.onSamples = onSamples;\n\n await parse();\n\n async function parse() {\n let cursor = 0;\n const maxReadSize = 30 * 1024 * 1024;\n while (true) {\n const data = (await reader.read(maxReadSize, {\n at: cursor,\n })) as MP4ArrayBuffer;\n if (data.byteLength === 0) break;\n data.fileStart = cursor;\n const nextPos = mp4boxFile.appendBuffer(data);\n if (nextPos == null) break;\n cursor = nextPos;\n }\n\n mp4boxFile.stop();\n }\n}\n","import { Log } from '@webav/internal-utils';\nimport { MP4Info, MP4Sample } from '@webav/mp4box.js';\nimport { file, tmpfile, write } from 'opfs-tools';\nimport { audioResample, extractPCM4AudioData, sleep } from '../av-utils';\nimport {\n extractFileConfig,\n quickParseMP4File,\n} from '../mp4-utils/mp4box-utils';\nimport { DEFAULT_AUDIO_CONF, IClip } from './iclip';\n\nlet CLIP_ID = 0;\n\ntype OPFSToolFile = ReturnType;\nfunction isOTFile(obj: any): obj is OPFSToolFile {\n return obj.kind === 'file' && obj.createReader instanceof Function;\n}\n\n// 用于内部创建 MP4Clip 实例\ntype MPClipCloneArgs = Awaited> & {\n localFile: OPFSToolFile;\n};\n\ninterface MP4DecoderConf {\n video: VideoDecoderConfig | null;\n audio: AudioDecoderConfig | null;\n}\n\ninterface MP4ClipOpts {\n audio?: boolean | { volume: number };\n /**\n * 不安全，随时可能废弃\n */\n __unsafe_hardwareAcceleration__?: HardwarePreference;\n}\n\ntype ExtMP4Sample = Omit & {\n is_idr: boolean;\n deleted?: boolean;\n data: null | Uint8Array;\n};\n\ntype LocalFileReader = Awaited>;\n\ntype ThumbnailOpts = {\n start: number;\n end: number;\n step: number;\n};\n\n/**\n * MP4 素材，解析 MP4 文件，使用 {@link MP4Clip.tick} 按需解码指定时间的图像帧\n *\n * 可用于实现视频抽帧、生成缩略图、视频编辑等功能\n *\n * @example\n * new MP4Clip((await fetch('')).body)\n * new MP4Clip(mp4File.stream())\n *\n * @see {@link Combinator}\n * @see [AVCanvas](../../av-canvas/classes/AVCanvas.html)\n *\n * @see [解码播放视频](https://webav-tech.github.io/WebAV/demo/1_1-decode-video)\n */\nexport class MP4Clip implements IClip {\n #insId = CLIP_ID++;\n\n #log = Log.create(`MP4Clip id:${this.#insId},`);\n\n ready: IClip['ready'];\n\n #destroyed = false;\n\n #meta = {\n // 微秒\n duration: 0,\n width: 0,\n height: 0,\n audioSampleRate: 0,\n audioChanCount: 0,\n };\n\n get meta() {\n return { ...this.#meta };\n }\n\n #localFile: OPFSToolFile;\n\n #headerBoxPos: Array<{ start: number; size: number }> = [];\n /**\n * 提供视频头（box: ftyp, moov）的二进制数据\n * 使用任意 mp4 demxer 解析即可获得详细的视频信息\n * 单元测试包含使用 mp4box.js 解析示例代码\n */\n async getFileHeaderBinData() {\n await this.ready;\n const oFile = await this.#localFile.getOriginFile();\n if (oFile == null) throw Error('MP4Clip localFile is not origin file');\n\n return await new Blob(\n this.#headerBoxPos.map(({ start, size }) =>\n oFile.slice(start, start + size),\n ),\n ).arrayBuffer();\n }\n\n #volume = 1;\n\n #videoSamples: ExtMP4Sample[] = [];\n\n #audioSamples: ExtMP4Sample[] = [];\n\n #videoFrameFinder: VideoFrameFinder | null = null;\n #audioFrameFinder: AudioFrameFinder | null = null;\n\n #decoderConf: {\n video: VideoDecoderConfig | null;\n audio: AudioDecoderConfig | null;\n } = {\n video: null,\n audio: null,\n };\n\n #opts: MP4ClipOpts = { audio: true };\n\n constructor(\n source: OPFSToolFile | ReadableStream | MPClipCloneArgs,\n opts: MP4ClipOpts = {},\n ) {\n if (\n !(source instanceof ReadableStream) &&\n !isOTFile(source) &&\n !Array.isArray(source.videoSamples)\n ) {\n throw Error('Illegal argument');\n }\n\n this.#opts = { audio: true, ...opts };\n this.#volume =\n typeof opts.audio === 'object' && 'volume' in opts.audio\n ? opts.audio.volume\n : 1;\n\n const initByStream = async (s: ReadableStream) => {\n await write(this.#localFile, s);\n return this.#localFile;\n };\n\n this.#localFile = isOTFile(source)\n ? source\n : 'localFile' in source\n ? source.localFile // from clone\n : tmpfile();\n\n this.ready = (\n source instanceof ReadableStream\n ? initByStream(source).then((otFile) =>\n mp4FileToSamples(otFile, this.#opts),\n )\n : isOTFile(source)\n ? mp4FileToSamples(source, this.#opts)\n : Promise.resolve(source)\n ).then(\n async ({ videoSamples, audioSamples, decoderConf, headerBoxPos }) => {\n this.#videoSamples = videoSamples;\n this.#audioSamples = audioSamples;\n this.#decoderConf = decoderConf;\n this.#headerBoxPos = headerBoxPos;\n\n const { videoFrameFinder, audioFrameFinder } = genDecoder(\n {\n video:\n decoderConf.video == null\n ? null\n : {\n ...decoderConf.video,\n hardwareAcceleration:\n this.#opts.__unsafe_hardwareAcceleration__,\n },\n audio: decoderConf.audio,\n },\n await this.#localFile.createReader(),\n videoSamples,\n audioSamples,\n this.#opts.audio !== false ? this.#volume : 0,\n );\n this.#videoFrameFinder = videoFrameFinder;\n this.#audioFrameFinder = audioFrameFinder;\n\n this.#meta = genMeta(decoderConf, videoSamples, audioSamples);\n this.#log.info('MP4Clip meta:', this.#meta);\n return { ...this.#meta };\n },\n );\n }\n\n /**\n * 拦截 {@link MP4Clip.tick} 方法返回的数据，用于对图像、音频数据二次处理\n * @param time 调用 tick 的时间\n * @param tickRet tick 返回的数据\n *\n * @see [移除视频绿幕背景](https://webav-tech.github.io/WebAV/demo/3_2-chromakey-video)\n */\n tickInterceptor: >>(\n time: number,\n tickRet: T,\n ) => Promise = async (_, tickRet) => tickRet;\n\n /**\n * 获取素材指定时刻的图像帧、音频数据\n * @param time 微秒\n */\n async tick(time: number): Promise<{\n video?: VideoFrame;\n audio: Float32Array[];\n state: 'success' | 'done';\n }> {\n if (time >= this.#meta.duration) {\n return await this.tickInterceptor(time, {\n audio: (await this.#audioFrameFinder?.find(time)) ?? [],\n state: 'done',\n });\n }\n\n const [audio, video] = await Promise.all([\n this.#audioFrameFinder?.find(time) ?? [],\n this.#videoFrameFinder?.find(time),\n ]);\n\n if (video == null) {\n return await this.tickInterceptor(time, {\n audio,\n state: 'success',\n });\n }\n\n return await this.tickInterceptor(time, {\n video,\n audio,\n state: 'success',\n });\n }\n\n #thumbAborter = new AbortController();\n /**\n * 生成缩略图，默认每个关键帧生成一个 100px 宽度的缩略图。\n *\n * @param imgWidth 缩略图宽度，默认 100\n * @param opts Partial\n * @returns Promise>\n */\n async thumbnails(\n imgWidth = 100,\n opts?: Partial,\n ): Promise> {\n this.#thumbAborter.abort();\n this.#thumbAborter = new AbortController();\n const aborterSignal = this.#thumbAborter.signal;\n\n await this.ready;\n const abortMsg = 'generate thumbnails aborted';\n if (aborterSignal.aborted) throw Error(abortMsg);\n\n const { width, height } = this.#meta;\n const convtr = createVF2BlobConvtr(\n imgWidth,\n Math.round(height * (imgWidth / width)),\n { quality: 0.1, type: 'image/png' },\n );\n\n return new Promise>(\n async (resolve, reject) => {\n let pngPromises: Array<{ ts: number; img: Promise }> = [];\n const vc = this.#decoderConf.video;\n if (vc == null || this.#videoSamples.length === 0) {\n resolver();\n return;\n }\n aborterSignal.addEventListener('abort', () => {\n reject(Error(abortMsg));\n });\n\n async function resolver() {\n if (aborterSignal.aborted) return;\n resolve(\n await Promise.all(\n pngPromises.map(async (it) => ({\n ts: it.ts,\n img: await it.img,\n })),\n ),\n );\n }\n\n function pushPngPromise(vf: VideoFrame) {\n pngPromises.push({\n ts: vf.timestamp,\n img: convtr(vf),\n });\n }\n\n const { start = 0, end = this.#meta.duration, step } = opts ?? {};\n if (step) {\n let cur = start;\n // 创建一个新的 VideoFrameFinder 实例，避免与 tick 方法共用而导致冲突\n const videoFrameFinder = new VideoFrameFinder(\n await this.#localFile.createReader(),\n this.#videoSamples,\n {\n ...vc,\n hardwareAcceleration: this.#opts.__unsafe_hardwareAcceleration__,\n },\n );\n while (cur <= end && !aborterSignal.aborted) {\n const vf = await videoFrameFinder.find(cur);\n if (vf) pushPngPromise(vf);\n cur += step;\n }\n videoFrameFinder.destroy();\n resolver();\n } else {\n await thumbnailByKeyFrame(\n this.#videoSamples,\n this.#localFile,\n vc,\n aborterSignal,\n { start, end },\n (vf, done) => {\n if (vf != null) pushPngPromise(vf);\n if (done) resolver();\n },\n );\n }\n },\n );\n }\n\n async split(time: number) {\n await this.ready;\n\n if (time <= 0 || time >= this.#meta.duration)\n throw Error('\"time\" out of bounds');\n\n const [preVideoSlice, postVideoSlice] = splitVideoSampleByTime(\n this.#videoSamples,\n time,\n );\n const [preAudioSlice, postAudioSlice] = splitAudioSampleByTime(\n this.#audioSamples,\n time,\n );\n const preClip = new MP4Clip(\n {\n localFile: this.#localFile,\n videoSamples: preVideoSlice ?? [],\n audioSamples: preAudioSlice ?? [],\n decoderConf: this.#decoderConf,\n headerBoxPos: this.#headerBoxPos,\n },\n this.#opts,\n );\n const postClip = new MP4Clip(\n {\n localFile: this.#localFile,\n videoSamples: postVideoSlice ?? [],\n audioSamples: postAudioSlice ?? [],\n decoderConf: this.#decoderConf,\n headerBoxPos: this.#headerBoxPos,\n },\n this.#opts,\n );\n await Promise.all([preClip.ready, postClip.ready]);\n\n return [preClip, postClip] as [this, this];\n }\n\n async clone() {\n await this.ready;\n const clip = new MP4Clip(\n {\n localFile: this.#localFile,\n videoSamples: [...this.#videoSamples],\n audioSamples: [...this.#audioSamples],\n decoderConf: this.#decoderConf,\n headerBoxPos: this.#headerBoxPos,\n },\n this.#opts,\n );\n await clip.ready;\n clip.tickInterceptor = this.tickInterceptor;\n return clip as this;\n }\n\n /**\n * 拆分 MP4Clip 为仅包含视频轨道和音频轨道的 MP4Clip\n * @returns Mp4CLip[]\n */\n async splitTrack() {\n await this.ready;\n const clips: MP4Clip[] = [];\n if (this.#videoSamples.length > 0) {\n const videoClip = new MP4Clip(\n {\n localFile: this.#localFile,\n videoSamples: [...this.#videoSamples],\n audioSamples: [],\n decoderConf: {\n video: this.#decoderConf.video,\n audio: null,\n },\n headerBoxPos: this.#headerBoxPos,\n },\n this.#opts,\n );\n await videoClip.ready;\n videoClip.tickInterceptor = this.tickInterceptor;\n clips.push(videoClip);\n }\n if (this.#audioSamples.length > 0) {\n const audioClip = new MP4Clip(\n {\n localFile: this.#localFile,\n videoSamples: [],\n audioSamples: [...this.#audioSamples],\n decoderConf: {\n audio: this.#decoderConf.audio,\n video: null,\n },\n headerBoxPos: this.#headerBoxPos,\n },\n this.#opts,\n );\n await audioClip.ready;\n audioClip.tickInterceptor = this.tickInterceptor;\n clips.push(audioClip);\n }\n\n return clips;\n }\n\n destroy(): void {\n if (this.#destroyed) return;\n this.#log.info('MP4Clip destroy');\n this.#destroyed = true;\n\n this.#videoFrameFinder?.destroy();\n this.#audioFrameFinder?.destroy();\n }\n}\n\nfunction genMeta(\n decoderConf: MP4DecoderConf,\n videoSamples: ExtMP4Sample[],\n audioSamples: ExtMP4Sample[],\n) {\n const meta = {\n duration: 0,\n width: 0,\n height: 0,\n audioSampleRate: 0,\n audioChanCount: 0,\n };\n if (decoderConf.video != null && videoSamples.length > 0) {\n meta.width = decoderConf.video.codedWidth ?? 0;\n meta.height = decoderConf.video.codedHeight ?? 0;\n }\n if (decoderConf.audio != null && audioSamples.length > 0) {\n meta.audioSampleRate = DEFAULT_AUDIO_CONF.sampleRate;\n meta.audioChanCount = DEFAULT_AUDIO_CONF.channelCount;\n }\n\n let vDuration = 0;\n let aDuration = 0;\n if (videoSamples.length > 0) {\n for (let i = videoSamples.length - 1; i >= 0; i--) {\n const s = videoSamples[i];\n if (s.deleted) continue;\n vDuration = s.cts + s.duration;\n break;\n }\n }\n if (audioSamples.length > 0) {\n const lastSampele = audioSamples.at(-1)!;\n aDuration = lastSampele.cts + lastSampele.duration;\n }\n meta.duration = Math.max(vDuration, aDuration);\n\n return meta;\n}\n\nfunction genDecoder(\n decoderConf: MP4DecoderConf,\n localFileReader: LocalFileReader,\n videoSamples: ExtMP4Sample[],\n audioSamples: ExtMP4Sample[],\n volume: number,\n) {\n return {\n audioFrameFinder:\n volume === 0 || decoderConf.audio == null || audioSamples.length === 0\n ? null\n : new AudioFrameFinder(\n localFileReader,\n audioSamples,\n decoderConf.audio,\n {\n volume,\n targetSampleRate: DEFAULT_AUDIO_CONF.sampleRate,\n },\n ),\n videoFrameFinder:\n decoderConf.video == null || videoSamples.length === 0\n ? null\n : new VideoFrameFinder(\n localFileReader,\n videoSamples,\n decoderConf.video,\n ),\n };\n}\n\nasync function mp4FileToSamples(otFile: OPFSToolFile, opts: MP4ClipOpts = {}) {\n let mp4Info: MP4Info | null = null;\n const decoderConf: MP4DecoderConf = { video: null, audio: null };\n let videoSamples: ExtMP4Sample[] = [];\n let audioSamples: ExtMP4Sample[] = [];\n let headerBoxPos: Array<{ start: number; size: number }> = [];\n\n let videoDeltaTS = -1;\n let audioDeltaTS = -1;\n let findedFirstSync = false;\n const reader = await otFile.createReader();\n await quickParseMP4File(\n reader,\n (data) => {\n mp4Info = data.info;\n const ftyp = data.mp4boxFile.ftyp!;\n headerBoxPos.push({ start: ftyp.start, size: ftyp.size });\n const moov = data.mp4boxFile.moov!;\n headerBoxPos.push({ start: moov.start, size: moov.size });\n\n let { videoDecoderConf: vc, audioDecoderConf: ac } = extractFileConfig(\n data.mp4boxFile,\n data.info,\n );\n decoderConf.video = vc ?? null;\n decoderConf.audio = ac ?? null;\n if (vc == null && ac == null) {\n Log.error('MP4Clip no video and audio track');\n }\n Log.info(\n 'mp4BoxFile moov ready',\n {\n ...data.info,\n tracks: null,\n videoTracks: null,\n audioTracks: null,\n },\n decoderConf,\n );\n },\n (_, type, samples) => {\n if (type === 'video') {\n if (videoDeltaTS === -1) videoDeltaTS = samples[0].dts;\n for (let i = 0; i < samples.length; i++) {\n const s = samples[i];\n if (!findedFirstSync && s.is_sync) {\n findedFirstSync = true;\n videoSamples.push(\n normalizeTimescale(s, videoDeltaTS, 'video', true),\n );\n } else {\n videoSamples.push(\n normalizeTimescale(s, videoDeltaTS, 'video', false),\n );\n }\n }\n } else if (type === 'audio' && opts.audio) {\n if (audioDeltaTS === -1) audioDeltaTS = samples[0].dts;\n for (const s of samples) {\n audioSamples.push(normalizeTimescale(s, audioDeltaTS, 'audio'));\n }\n }\n },\n );\n await reader.close();\n\n const lastSampele = videoSamples.at(-1) ?? audioSamples.at(-1);\n if (mp4Info == null) {\n throw Error('MP4Clip stream is done, but not emit ready');\n } else if (lastSampele == null) {\n throw Error('MP4Clip stream not contain any sample');\n }\n // 修复首帧黑帧\n fixFirstBlackFrame(videoSamples);\n Log.info('mp4 stream parsed');\n return {\n videoSamples,\n audioSamples,\n decoderConf,\n headerBoxPos,\n };\n}\n\nfunction normalizeTimescale(\n s: MP4Sample,\n delta = 0,\n sampleType: 'video' | 'audio',\n isFirstSync?: boolean,\n) {\n // todo: perf 丢弃多余字段，小尺寸对象性能更好\n let offset = s.offset;\n const isVideoSync = sampleType === 'video' && s.is_sync;\n const idrOffset = isVideoSync\n ? idrNALUOffset(s.data, s.description.type, offset)\n : -1;\n\n // 默认信任第一个关键帧 是 IDR 帧，兼容某些异常标注的视频文件\n let is_idr = isFirstSync === true && isVideoSync;\n let size = s.size;\n if (idrOffset >= 0) {\n // 当 IDR 帧前面包含非图像帧数据（如 SEI），可能导致解码失败\n // 所以此处通过控制 offset、size 字段 跳过非图像帧数据\n offset = idrOffset;\n size -= idrOffset - offset;\n // 非第一个关键帧，如果根据 naluType 判定是 IDR 帧，则设置 is_idr\n is_idr = true;\n }\n\n return {\n ...s,\n is_idr,\n offset,\n size,\n cts: ((s.cts - delta) / s.timescale) * 1e6,\n dts: ((s.dts - delta) / s.timescale) * 1e6,\n duration: (s.duration / s.timescale) * 1e6,\n timescale: 1e6,\n // 音频数据量可控，直接保存在内存中\n data: sampleType === 'video' ? null : s.data,\n };\n}\n\nclass VideoFrameFinder {\n #dec: VideoDecoder | null = null;\n constructor(\n public localFileReader: LocalFileReader,\n public samples: ExtMP4Sample[],\n public conf: VideoDecoderConfig,\n ) {}\n\n #ts = 0;\n #curAborter = { abort: false, st: performance.now() };\n find = async (time: number): Promise => {\n if (\n this.#dec == null ||\n this.#dec.state === 'closed' ||\n time <= this.#ts ||\n time - this.#ts > 3e6\n ) {\n this.#reset(time);\n }\n\n this.#curAborter.abort = true;\n this.#ts = time;\n\n this.#curAborter = { abort: false, st: performance.now() };\n const vf = await this.#parseFrame(time, this.#dec, this.#curAborter);\n this.#sleepCnt = 0;\n return vf;\n };\n\n // fix VideoFrame duration is null\n #lastVfDur = 0;\n\n #downgradeSoftDecode = false;\n #videoDecCusorIdx = 0;\n #videoFrames: VideoFrame[] = [];\n #outputFrameCnt = 0;\n #inputChunkCnt = 0;\n #sleepCnt = 0;\n #predecodeErr = false;\n #parseFrame = async (\n time: number,\n dec: VideoDecoder | null,\n aborter: { abort: boolean; st: number },\n ): Promise => {\n if (dec == null || dec.state === 'closed' || aborter.abort) return null;\n\n if (this.#videoFrames.length > 0) {\n const vf = this.#videoFrames[0];\n if (time < vf.timestamp) return null;\n // 弹出第一帧\n this.#videoFrames.shift();\n // 第一帧过期，找下一帧\n if (time > vf.timestamp + (vf.duration ?? 0)) {\n vf.close();\n return await this.#parseFrame(time, dec, aborter);\n }\n\n if (!this.#predecodeErr && this.#videoFrames.length < 10) {\n // 预解码 避免等待\n this.#startDecode(dec).catch((err) => {\n this.#predecodeErr = true;\n this.#reset(time);\n throw err;\n });\n }\n // 符合期望\n return vf;\n }\n\n // 缺少帧数据\n if (\n this.#decoding ||\n (this.#outputFrameCnt < this.#inputChunkCnt && dec.decodeQueueSize > 0)\n ) {\n if (performance.now() - aborter.st > 6e3) {\n throw Error(\n `MP4Clip.tick video timeout, ${JSON.stringify(this.#getState())}`,\n );\n }\n // 解码中，等待，然后重试\n this.#sleepCnt += 1;\n await sleep(15);\n } else if (this.#videoDecCusorIdx >= this.samples.length) {\n // decode completed\n return null;\n } else {\n try {\n await this.#startDecode(dec);\n } catch (err) {\n this.#reset(time);\n throw err;\n }\n }\n return await this.#parseFrame(time, dec, aborter);\n };\n\n #decoding = false;\n #startDecode = async (dec: VideoDecoder) => {\n if (this.#decoding || dec.decodeQueueSize > 600) return;\n\n // 启动解码任务，然后重试\n let endIdx = this.#videoDecCusorIdx + 1;\n if (endIdx > this.samples.length) return;\n\n this.#decoding = true;\n // 该 GoP 时间区间有时间匹配，且未被删除的帧\n let hasValidFrame = false;\n for (; endIdx < this.samples.length; endIdx++) {\n const s = this.samples[endIdx];\n if (!hasValidFrame && !s.deleted) {\n hasValidFrame = true;\n }\n // 找一个 GoP，所以是下一个 IDR 帧结束\n if (s.is_idr) break;\n }\n\n if (hasValidFrame) {\n const samples = this.samples.slice(this.#videoDecCusorIdx, endIdx);\n if (samples[0]?.is_idr !== true) {\n Log.warn('First sample not idr frame');\n } else {\n const readStarTime = performance.now();\n const chunks = await videosamples2Chunks(samples, this.localFileReader);\n\n const readCost = performance.now() - readStarTime;\n if (readCost > 1000) {\n const first = samples[0];\n const last = samples.at(-1)!;\n const rangSize = last.offset + last.size - first.offset;\n Log.warn(\n `Read video samples time cost: ${Math.round(readCost)}ms, file chunk size: ${rangSize}`,\n );\n }\n // Wait for the previous asynchronous operation to complete, at which point the task may have already been terminated\n if (dec.state === 'closed') return;\n\n this.#lastVfDur = chunks[0]?.duration ?? 0;\n decodeGoP(dec, chunks, {\n onDecodingError: (err) => {\n if (this.#downgradeSoftDecode) {\n throw err;\n } else if (this.#outputFrameCnt === 0) {\n this.#downgradeSoftDecode = true;\n Log.warn('Downgrade to software decode');\n this.#reset();\n }\n },\n });\n\n this.#inputChunkCnt += chunks.length;\n }\n }\n this.#videoDecCusorIdx = endIdx;\n this.#decoding = false;\n };\n\n #reset = (time?: number) => {\n this.#decoding = false;\n this.#videoFrames.forEach((f) => f.close());\n this.#videoFrames = [];\n if (time == null || time === 0) {\n this.#videoDecCusorIdx = 0;\n } else {\n let keyIdx = 0;\n for (let i = 0; i < this.samples.length; i++) {\n const s = this.samples[i];\n if (s.is_idr) keyIdx = i;\n if (s.cts < time) continue;\n this.#videoDecCusorIdx = keyIdx;\n break;\n }\n }\n this.#inputChunkCnt = 0;\n this.#outputFrameCnt = 0;\n if (this.#dec?.state !== 'closed') this.#dec?.close();\n const encoderConf = {\n ...this.conf,\n ...(this.#downgradeSoftDecode\n ? { hardwareAcceleration: 'prefer-software' }\n : {}),\n } as VideoDecoderConfig;\n this.#dec = new VideoDecoder({\n output: (vf) => {\n this.#outputFrameCnt += 1;\n if (vf.timestamp === -1) {\n vf.close();\n return;\n }\n let rsVf = vf;\n if (vf.duration == null) {\n rsVf = new VideoFrame(vf, {\n duration: this.#lastVfDur,\n });\n vf.close();\n }\n this.#videoFrames.push(rsVf);\n },\n error: (err) => {\n if (err.message.includes('Codec reclaimed due to inactivity')) {\n // todo: 因无活动被自动关闭的解码器，是否需要自动重启？\n this.#dec = null;\n Log.warn(err.message);\n return;\n }\n\n const errMsg = `VideoFinder VideoDecoder err: ${err.message}, config: ${JSON.stringify(encoderConf)}, state: ${JSON.stringify(this.#getState())}`;\n Log.error(errMsg);\n throw Error(errMsg);\n },\n });\n this.#dec.configure(encoderConf);\n };\n\n #getState = () => ({\n time: this.#ts,\n decState: this.#dec?.state,\n decQSize: this.#dec?.decodeQueueSize,\n decCusorIdx: this.#videoDecCusorIdx,\n sampleLen: this.samples.length,\n inputCnt: this.#inputChunkCnt,\n outputCnt: this.#outputFrameCnt,\n cacheFrameLen: this.#videoFrames.length,\n softDeocde: this.#downgradeSoftDecode,\n clipIdCnt: CLIP_ID,\n sleepCnt: this.#sleepCnt,\n memInfo: memoryUsageInfo(),\n });\n\n destroy = () => {\n if (this.#dec?.state !== 'closed') this.#dec?.close();\n this.#dec = null;\n this.#curAborter.abort = true;\n this.#videoFrames.forEach((f) => f.close());\n this.#videoFrames = [];\n this.localFileReader.close();\n };\n}\n\nfunction findIndexOfSamples(time: number, samples: ExtMP4Sample[]) {\n for (let i = 0; i < samples.length; i++) {\n const s = samples[i];\n if (time >= s.cts && time < s.cts + s.duration) {\n return i;\n }\n if (s.cts > time) break;\n }\n return 0;\n}\n\nclass AudioFrameFinder {\n #volume = 1;\n #sampleRate;\n constructor(\n public localFileReader: LocalFileReader,\n public samples: ExtMP4Sample[],\n public conf: AudioDecoderConfig,\n opts: { volume: number; targetSampleRate: number },\n ) {\n this.#volume = opts.volume;\n this.#sampleRate = opts.targetSampleRate;\n }\n\n #dec: ReturnType | null = null;\n #curAborter = { abort: false, st: performance.now() };\n find = async (time: number): Promise => {\n const needResetTime = time <= this.#ts || time - this.#ts > 0.1e6;\n if (this.#dec == null || this.#dec.state === 'closed' || needResetTime) {\n this.#reset();\n }\n\n if (needResetTime) {\n // 前后获取音频数据差异不能超过 100ms(经验值)，否则视为 seek 操作，重置解码器\n // seek 操作，重置时间\n this.#ts = time;\n this.#decCusorIdx = findIndexOfSamples(time, this.samples);\n }\n\n this.#curAborter.abort = true;\n const deltaTime = time - this.#ts;\n this.#ts = time;\n\n this.#curAborter = { abort: false, st: performance.now() };\n\n const pcmData = await this.#parseFrame(\n Math.ceil(deltaTime * (this.#sampleRate / 1e6)),\n this.#dec,\n this.#curAborter,\n );\n this.#sleepCnt = 0;\n return pcmData;\n };\n\n #ts = 0;\n #decCusorIdx = 0;\n #pcmData: {\n frameCnt: number;\n data: [Float32Array, Float32Array][];\n } = {\n frameCnt: 0,\n data: [],\n };\n #sleepCnt = 0;\n #parseFrame = async (\n emitFrameCnt: number,\n dec: ReturnType | null = null,\n aborter: { abort: boolean; st: number },\n ): Promise => {\n if (\n dec == null ||\n aborter.abort ||\n dec.state === 'closed' ||\n emitFrameCnt === 0\n ) {\n return [];\n }\n\n // 数据满足需要\n const ramainFrameCnt = this.#pcmData.frameCnt - emitFrameCnt;\n if (ramainFrameCnt > 0) {\n // 剩余音频数据小于 100ms，预先解码\n if (ramainFrameCnt < DEFAULT_AUDIO_CONF.sampleRate / 10) {\n this.#startDecode(dec);\n }\n return emitAudioFrames(this.#pcmData, emitFrameCnt);\n }\n\n if (dec.decoding) {\n if (performance.now() - aborter.st > 3e3) {\n aborter.abort = true;\n throw Error(\n `MP4Clip.tick audio timeout, ${JSON.stringify(this.#getState())}`,\n );\n }\n // 解码中，等待\n this.#sleepCnt += 1;\n await sleep(15);\n } else if (this.#decCusorIdx >= this.samples.length - 1) {\n // 最后片段，返回剩余数据\n return emitAudioFrames(this.#pcmData, this.#pcmData.frameCnt);\n } else {\n this.#startDecode(dec);\n }\n return this.#parseFrame(emitFrameCnt, dec, aborter);\n };\n\n #startDecode = (dec: ReturnType) => {\n const onceDecodeCnt = 10;\n if (dec.decodeQueueSize > onceDecodeCnt) return;\n // 启动解码任务\n const samples = [];\n let i = this.#decCusorIdx;\n while (i < this.samples.length) {\n const s = this.samples[i];\n i += 1;\n if (s.deleted) continue;\n samples.push(s);\n if (samples.length >= onceDecodeCnt) break;\n }\n this.#decCusorIdx = i;\n\n dec.decode(\n samples.map(\n (s) =>\n new EncodedAudioChunk({\n type: 'key',\n timestamp: s.cts,\n duration: s.duration,\n data: s.data!,\n }),\n ),\n );\n };\n\n #reset = () => {\n this.#ts = 0;\n this.#decCusorIdx = 0;\n this.#pcmData = {\n frameCnt: 0,\n data: [],\n };\n this.#dec?.close();\n this.#dec = createAudioChunksDecoder(\n this.conf,\n {\n resampleRate: DEFAULT_AUDIO_CONF.sampleRate,\n volume: this.#volume,\n },\n (pcmArr) => {\n this.#pcmData.data.push(pcmArr as [Float32Array, Float32Array]);\n this.#pcmData.frameCnt += pcmArr[0].length;\n },\n );\n };\n\n #getState = () => ({\n time: this.#ts,\n decState: this.#dec?.state,\n decQSize: this.#dec?.decodeQueueSize,\n decCusorIdx: this.#decCusorIdx,\n sampleLen: this.samples.length,\n pcmLen: this.#pcmData.frameCnt,\n clipIdCnt: CLIP_ID,\n sleepCnt: this.#sleepCnt,\n memInfo: memoryUsageInfo(),\n });\n\n destroy = () => {\n this.#dec = null;\n this.#curAborter.abort = true;\n this.#pcmData = {\n frameCnt: 0,\n data: [],\n };\n this.localFileReader.close();\n };\n}\n\nfunction createAudioChunksDecoder(\n decoderConf: AudioDecoderConfig,\n opts: { resampleRate: number; volume: number },\n outputCb: (pcm: Float32Array[]) => void,\n) {\n let inputCnt = 0;\n let outputCnt = 0;\n const outputHandler = (pcmArr: Float32Array[]) => {\n outputCnt += 1;\n if (pcmArr.length === 0) return;\n // 音量调节\n if (opts.volume !== 1) {\n for (const pcm of pcmArr)\n for (let i = 0; i < pcm.length; i++) pcm[i] *= opts.volume;\n }\n\n // 补齐双声道\n if (pcmArr.length === 1) pcmArr = [pcmArr[0], pcmArr[0]];\n\n outputCb(pcmArr);\n };\n const resampleQ = createPromiseQueue(outputHandler);\n\n const needResample = opts.resampleRate !== decoderConf.sampleRate;\n let adec = new AudioDecoder({\n output: (ad) => {\n const pcm = extractPCM4AudioData(ad);\n if (needResample) {\n resampleQ(() =>\n audioResample(pcm, ad.sampleRate, {\n rate: opts.resampleRate,\n chanCount: ad.numberOfChannels,\n }),\n );\n } else {\n outputHandler(pcm);\n }\n ad.close();\n },\n error: (err) => {\n if (err.message.includes('Codec reclaimed due to inactivity')) {\n return;\n }\n handleDecodeError('MP4Clip AudioDecoder err', err as Error);\n },\n });\n adec.configure(decoderConf);\n\n function handleDecodeError(prefixStr: string, err: Error) {\n const errMsg = `${prefixStr}: ${(err as Error).message}, state: ${JSON.stringify(\n {\n qSize: adec.decodeQueueSize,\n state: adec.state,\n inputCnt,\n outputCnt,\n },\n )}`;\n Log.error(errMsg);\n throw Error(errMsg);\n }\n\n return {\n decode(chunks: EncodedAudioChunk[]) {\n inputCnt += chunks.length;\n try {\n for (const chunk of chunks) adec.decode(chunk);\n } catch (err) {\n handleDecodeError('decode audio chunk error', err as Error);\n }\n },\n close() {\n if (adec.state !== 'closed') adec.close();\n },\n get decoding() {\n return inputCnt > outputCnt && adec.decodeQueueSize > 0;\n },\n get state() {\n return adec.state;\n },\n get decodeQueueSize() {\n return adec.decodeQueueSize;\n },\n };\n}\n\n// 并行执行任务，但按顺序emit结果\nfunction createPromiseQueue(onResult: (data: T) => void) {\n const rsCache: T[] = [];\n let waitingIdx = 0;\n\n function updateRs(rs: T, emitIdx: number) {\n rsCache[emitIdx] = rs;\n emitRs();\n }\n\n function emitRs() {\n const rs = rsCache[waitingIdx];\n if (rs == null) return;\n onResult(rs);\n\n waitingIdx += 1;\n emitRs();\n }\n\n let addIdx = 0;\n return (task: () => Promise) => {\n const emitIdx = addIdx;\n addIdx += 1;\n task()\n .then((rs) => updateRs(rs, emitIdx))\n .catch((err) => updateRs(err, emitIdx));\n };\n}\n\nfunction emitAudioFrames(\n pcmData: { frameCnt: number; data: [Float32Array, Float32Array][] },\n emitCnt: number,\n) {\n // todo: perf 重复利用内存空间\n const audio = [new Float32Array(emitCnt), new Float32Array(emitCnt)];\n let offset = 0;\n let i = 0;\n for (; i < pcmData.data.length; ) {\n const [chan0, chan1] = pcmData.data[i];\n if (offset + chan0.length > emitCnt) {\n const gapCnt = emitCnt - offset;\n audio[0].set(chan0.subarray(0, gapCnt), offset);\n audio[1].set(chan1.subarray(0, gapCnt), offset);\n pcmData.data[i][0] = chan0.subarray(gapCnt, chan0.length);\n pcmData.data[i][1] = chan1.subarray(gapCnt, chan1.length);\n break;\n } else {\n audio[0].set(chan0, offset);\n audio[1].set(chan1, offset);\n offset += chan0.length;\n i++;\n }\n }\n pcmData.data = pcmData.data.slice(i);\n pcmData.frameCnt -= emitCnt;\n return audio;\n}\n\nasync function videosamples2Chunks(\n samples: ExtMP4Sample[],\n reader: Awaited>,\n): Promise {\n const first = samples[0];\n const last = samples.at(-1);\n if (last == null) return [];\n\n const rangSize = last.offset + last.size - first.offset;\n if (rangSize < 30e6) {\n // 单次读取数据小于 30M，就一次性读取数据，降低 IO 频次\n const data = new Uint8Array(\n await reader.read(rangSize, { at: first.offset }),\n );\n return samples.map((s) => {\n const offset = s.offset - first.offset;\n return new EncodedVideoChunk({\n type: s.is_sync ? 'key' : 'delta',\n timestamp: s.cts,\n duration: s.duration,\n data: data.subarray(offset, offset + s.size),\n });\n });\n }\n\n return await Promise.all(\n samples.map(async (s) => {\n return new EncodedVideoChunk({\n type: s.is_sync ? 'key' : 'delta',\n timestamp: s.cts,\n duration: s.duration,\n data: await reader.read(s.size, {\n at: s.offset,\n }),\n });\n }),\n );\n}\n\nfunction createVF2BlobConvtr(\n width: number,\n height: number,\n opts?: ImageEncodeOptions,\n) {\n const cvs = new OffscreenCanvas(width, height);\n const ctx = cvs.getContext('2d')!;\n\n return async (vf: VideoFrame) => {\n ctx.drawImage(vf, 0, 0, width, height);\n vf.close();\n const blob = await cvs.convertToBlob(opts);\n return blob;\n };\n}\n\nfunction splitVideoSampleByTime(videoSamples: ExtMP4Sample[], time: number) {\n if (videoSamples.length === 0) return [];\n let gopStartIdx = 0;\n let gopEndIdx = 0;\n let hitIdx = -1;\n for (let i = 0; i < videoSamples.length; i++) {\n const s = videoSamples[i];\n if (hitIdx === -1 && time < s.cts) hitIdx = i - 1;\n if (s.is_idr) {\n if (hitIdx === -1) {\n gopStartIdx = i;\n } else {\n gopEndIdx = i;\n break;\n }\n }\n }\n\n const hitSample = videoSamples[hitIdx];\n if (hitSample == null) throw Error('Not found video sample by time');\n\n const preSlice = videoSamples\n .slice(0, gopEndIdx === 0 ? videoSamples.length : gopEndIdx)\n .map((s) => ({ ...s }));\n for (let i = gopStartIdx; i < preSlice.length; i++) {\n const s = preSlice[i];\n if (time < s.cts) {\n s.deleted = true;\n s.cts = -1;\n }\n }\n fixFirstBlackFrame(preSlice);\n\n const postSlice = videoSamples\n .slice(hitSample.is_idr ? hitIdx : gopStartIdx)\n .map((s) => ({ ...s, cts: s.cts - time }));\n\n for (const s of postSlice) {\n if (s.cts < 0) {\n s.deleted = true;\n s.cts = -1;\n }\n }\n fixFirstBlackFrame(postSlice);\n\n return [preSlice, postSlice];\n}\n\nfunction splitAudioSampleByTime(audioSamples: ExtMP4Sample[], time: number) {\n if (audioSamples.length === 0) return [];\n let hitIdx = -1;\n for (let i = 0; i < audioSamples.length; i++) {\n const s = audioSamples[i];\n if (time > s.cts) continue;\n hitIdx = i;\n break;\n }\n if (hitIdx === -1) throw Error('Not found audio sample by time');\n const preSlice = audioSamples.slice(0, hitIdx).map((s) => ({ ...s }));\n const postSlice = audioSamples\n .slice(hitIdx)\n .map((s) => ({ ...s, cts: s.cts - time }));\n return [preSlice, postSlice];\n}\n\n// 兼容解码错误\nfunction decodeGoP(\n dec: VideoDecoder,\n chunks: EncodedVideoChunk[],\n opts: {\n onDecodingError?: (err: Error) => void;\n },\n) {\n if (dec.state !== 'configured') return;\n for (let i = 0; i < chunks.length; i++) dec.decode(chunks[i]);\n\n // todo：flush 之后下一帧必须是 IDR 帧，是否可以根据情况再决定调用 flush？\n // windows 某些设备 flush 可能不会被 resolved，所以不能 await flush\n dec.flush().catch((err) => {\n if (!(err instanceof Error)) throw err;\n if (\n err.message.includes('Decoding error') &&\n opts.onDecodingError != null\n ) {\n opts.onDecodingError(err);\n return;\n }\n // reset 中断解码器，预期会抛出 AbortedError\n if (!err.message.includes('Aborted due to close')) {\n throw err;\n }\n });\n}\n\nfunction idrNALUOffset(\n u8Arr: Uint8Array,\n type: MP4Sample['description']['type'],\n startOffset: number,\n) {\n if (type !== 'avc1' && type !== 'hvc1') return 0;\n\n const dv = new DataView(u8Arr.buffer);\n let i = startOffset;\n for (; i < u8Arr.byteLength - 4; ) {\n if (type === 'avc1' && (dv.getUint8(i + 4) & 0x1f) === 5) {\n return i;\n } else if (type === 'hvc1') {\n const nalUnitType = (dv.getUint8(i + 4) >> 1) & 0x3f;\n if (nalUnitType === 19 || nalUnitType === 20) return i;\n }\n // 跳至下一个 NALU 继续检查\n i += dv.getUint32(i) + 4;\n }\n return -1;\n}\n\nasync function thumbnailByKeyFrame(\n samples: ExtMP4Sample[],\n localFile: OPFSToolFile,\n decConf: VideoDecoderConfig,\n abortSingl: AbortSignal,\n time: { start: number; end: number },\n onOutput: (vf: VideoFrame | null, done: boolean) => void,\n) {\n const fileReader = await localFile.createReader();\n\n const chunks = await videosamples2Chunks(\n samples.filter(\n (s) =>\n !s.deleted && s.is_sync && s.cts >= time.start && s.cts <= time.end,\n ),\n fileReader,\n );\n if (chunks.length === 0 || abortSingl.aborted) return;\n\n let outputCnt = 0;\n decodeGoP(createVideoDec(), chunks, {\n onDecodingError: (err) => {\n Log.warn('thumbnailsByKeyFrame', err);\n // 尝试降级一次\n if (outputCnt === 0) {\n decodeGoP(createVideoDec(true), chunks, {\n onDecodingError: (err) => {\n fileReader.close();\n Log.error('thumbnailsByKeyFrame retry soft deocde', err);\n },\n });\n } else {\n onOutput(null, true);\n fileReader.close();\n }\n },\n });\n\n function createVideoDec(downgrade = false) {\n const encoderConf = {\n ...decConf,\n ...(downgrade ? { hardwareAcceleration: 'prefer-software' } : {}),\n } as VideoDecoderConfig;\n const dec = new VideoDecoder({\n output: (vf) => {\n outputCnt += 1;\n const done = outputCnt === chunks.length;\n onOutput(vf, done);\n if (done) {\n fileReader.close();\n if (dec.state !== 'closed') dec.close();\n }\n },\n error: (err) => {\n const errMsg = `thumbnails decoder error: ${err.message}, config: ${JSON.stringify(encoderConf)}, state: ${JSON.stringify(\n {\n qSize: dec.decodeQueueSize,\n state: dec.state,\n outputCnt,\n inputCnt: chunks.length,\n },\n )}`;\n Log.error(errMsg);\n throw Error(errMsg);\n },\n });\n abortSingl.addEventListener('abort', () => {\n fileReader.close();\n if (dec.state !== 'closed') dec.close();\n });\n dec.configure(encoderConf);\n return dec;\n }\n}\n\n// 如果第一帧出现的时间偏移较大，会导致第一帧为黑帧，这里尝试自动消除第一帧前的黑帧\nfunction fixFirstBlackFrame(samples: ExtMP4Sample[]) {\n let iframeCnt = 0;\n let minCtsSample: ExtMP4Sample | null = null;\n // cts 最小表示视频的第一帧\n for (const s of samples) {\n if (s.deleted) continue;\n // 最多检测两个 I 帧之间的帧\n if (s.is_sync) iframeCnt += 1;\n if (iframeCnt >= 2) break;\n\n if (minCtsSample == null || s.cts < minCtsSample.cts) {\n minCtsSample = s;\n }\n }\n // 200ms 是经验值，自动消除 200ms 内的黑帧，超过则不处理\n if (minCtsSample != null && minCtsSample.cts < 200e3) {\n minCtsSample.duration += minCtsSample.cts;\n minCtsSample.cts = 0;\n }\n}\n\nfunction memoryUsageInfo() {\n try {\n // @ts-ignore\n const mem = performance.memory;\n return {\n jsHeapSizeLimit: mem.jsHeapSizeLimit,\n totalJSHeapSize: mem.totalJSHeapSize,\n usedJSHeapSize: mem.usedJSHeapSize,\n percentUsed: (mem.usedJSHeapSize / mem.jsHeapSizeLimit).toFixed(3),\n percentTotal: (mem.totalJSHeapSize / mem.jsHeapSizeLimit).toFixed(3),\n };\n } catch (err) {\n return {};\n }\n}\n\nif (import.meta.vitest) {\n const { it, expect } = import.meta.vitest;\n\n it('normalizeTimescale should compatible with anomalous data', () => {\n // 视频容器数据异常时，直接使用容器的信息\n // 找不到 IDR 帧时，直接信任容器的 offset、size\n const s: MP4Sample = {\n offset: 48,\n size: 1000,\n cts: 0,\n dts: 0,\n duration: 1e6,\n timescale: 1000,\n is_sync: true,\n deleted: false,\n // 异常的数据，offset = 48 时，并非 idr 帧，且 nalu 长度取值错误\n data: new Uint8Array([\n 0, 0, 0, 21, 103, 100, 0, 40, 172, 43, 32, 15, 0, 68, 216, 8, 128, 0, 1,\n 244, 0, 0, 93, 192, 66, 0, 0, 0, 4, 104, 235, 143, 44, 0, 1, 134, 130,\n 101, 184, 4, 127, 135, 242, 7, 227, 27, 39, 154, 66, 5, 153, 27, 242,\n 78, 0, 168, 62, 16, 51, 168, 150, 136, 69, 7, 149, 83, 39, 27, 153, 130,\n 214, 70, 144, 194, 191, 167, 74, 7, 11, 97, 224, 27, 193, 2, 122, 234,\n 71, 95, 7, 192, 199, 28, 181, 215, 12, 201, 182, 43, 105, 91,\n ]),\n track_id: 1,\n // @ts-ignore\n description: { type: 'avc1' },\n is_rap: false,\n };\n const normalized = normalizeTimescale(s, 0, 'video', true);\n expect(normalized.offset).toBe(48);\n expect(normalized.size).toBe(1000);\n expect(normalized.is_sync).toBe(normalized.is_idr);\n });\n}\n","import { Log } from '@webav/internal-utils';\nimport { decodeImg } from '../av-utils';\nimport { IClip } from './iclip';\n\ntype AnimateImgType = 'avif' | 'webp' | 'png' | 'gif';\n\n/**\n * 图像素材，支持动图\n *\n * 普通文字可通过 {@link renderTxt2ImgBitmap} 转换成图片素材\n *\n * @example\n * new ImgClip((await fetch('')).body);\n *\n * @example\n * new ImgClip(\n * await renderTxt2ImgBitmap(\n * '水印',\n * `font-size:40px; color: white; text-shadow: 2px 2px 6px red;`,\n * )\n * )\n *\n * @see [视频合成](https://webav-tech.github.io/WebAV/demo/2_1-concat-video)\n */\nexport class ImgClip implements IClip {\n ready: IClip['ready'];\n\n #meta = {\n // 微秒\n duration: 0,\n width: 0,\n height: 0,\n };\n\n /**\n * ⚠️ 静态图片的 duration 为 Infinity\n *\n * 使用 Sprite 包装时需要将它的 duration 设置为有限数\n *\n */\n get meta() {\n return { ...this.#meta };\n }\n\n #img: ImageBitmap | null = null;\n\n #frames: VideoFrame[] = [];\n\n /**\n * 静态图片可使用流、ImageBitmap 初始化\n *\n * 动图需要使用 VideoFrame[] 或提供图片类型\n */\n constructor(\n dataSource:\n | ReadableStream\n | ImageBitmap\n | VideoFrame[]\n | { type: `image/${AnimateImgType}`; stream: ReadableStream },\n ) {\n const initWithImgBitmap = (imgBitmap: ImageBitmap) => {\n this.#img = imgBitmap;\n this.#meta.width = imgBitmap.width;\n this.#meta.height = imgBitmap.height;\n this.#meta.duration = Infinity;\n return { ...this.#meta };\n };\n\n if (dataSource instanceof ReadableStream) {\n this.ready = new Response(dataSource)\n .blob()\n .then((data) => createImageBitmap(data))\n .then(initWithImgBitmap);\n } else if (dataSource instanceof ImageBitmap) {\n this.ready = Promise.resolve(initWithImgBitmap(dataSource));\n } else if (\n Array.isArray(dataSource) &&\n dataSource.every((it) => it instanceof VideoFrame)\n ) {\n this.#frames = dataSource;\n const frame = this.#frames[0];\n if (frame == null) throw Error('The frame count must be greater than 0');\n this.#meta = {\n width: frame.displayWidth,\n height: frame.displayHeight,\n duration: this.#frames.reduce(\n (acc, cur) => acc + (cur.duration ?? 0),\n 0,\n ),\n };\n this.ready = Promise.resolve({ ...this.#meta, duration: Infinity });\n } else if ('type' in dataSource) {\n this.ready = this.#initAnimateImg(\n dataSource.stream,\n dataSource.type,\n ).then(() => ({\n width: this.#meta.width,\n height: this.#meta.height,\n duration: Infinity,\n }));\n } else {\n throw Error('Illegal arguments');\n }\n }\n\n async #initAnimateImg(\n stream: ReadableStream,\n type: `image/${AnimateImgType}`,\n ) {\n this.#frames = await decodeImg(stream, type);\n const firstVf = this.#frames[0];\n if (firstVf == null) throw Error('No frame available in gif');\n\n this.#meta = {\n duration: this.#frames.reduce((acc, cur) => acc + (cur.duration ?? 0), 0),\n width: firstVf.codedWidth,\n height: firstVf.codedHeight,\n };\n Log.info('ImgClip ready:', this.#meta);\n }\n\n tickInterceptor: >>(\n time: number,\n tickRet: T,\n ) => Promise = async (_, tickRet) => tickRet;\n\n async tick(time: number): Promise<{\n video: ImageBitmap | VideoFrame;\n state: 'success';\n }> {\n if (this.#img != null) {\n return await this.tickInterceptor(time, {\n video: await createImageBitmap(this.#img),\n state: 'success',\n });\n }\n const tt = time % this.#meta.duration;\n return await this.tickInterceptor(time, {\n video: (\n this.#frames.find(\n (f) => tt >= f.timestamp && tt <= f.timestamp + (f.duration ?? 0),\n ) ?? this.#frames[0]\n ).clone(),\n state: 'success',\n });\n }\n\n async split(time: number) {\n await this.ready;\n if (this.#img != null) {\n return [\n new ImgClip(await createImageBitmap(this.#img)),\n new ImgClip(await createImageBitmap(this.#img)),\n ] as [this, this];\n }\n let hitIdx = -1;\n for (let i = 0; i < this.#frames.length; i++) {\n const vf = this.#frames[i];\n if (time > vf.timestamp) continue;\n hitIdx = i;\n break;\n }\n if (hitIdx === -1) throw Error('Not found frame by time');\n const preSlice = this.#frames\n .slice(0, hitIdx)\n .map((vf) => new VideoFrame(vf));\n const postSlice = this.#frames.slice(hitIdx).map(\n (vf) =>\n new VideoFrame(vf, {\n timestamp: vf.timestamp - time,\n }),\n );\n return [new ImgClip(preSlice), new ImgClip(postSlice)] as [this, this];\n }\n\n async clone() {\n await this.ready;\n const data =\n this.#img == null\n ? this.#frames.map((vf) => vf.clone())\n : await createImageBitmap(this.#img);\n const newClip = new ImgClip(data) as this;\n newClip.tickInterceptor = this.tickInterceptor;\n return newClip;\n }\n\n destroy(): void {\n Log.info('ImgClip destroy');\n this.#img?.close();\n this.#frames.forEach((f) => f.close());\n }\n}\n","import { Log } from '@webav/internal-utils';\nimport {\n concatPCMFragments,\n extractPCM4AudioBuffer,\n ringSliceFloat32Array,\n} from '../av-utils';\nimport { DEFAULT_AUDIO_CONF, IClip } from './iclip';\n\ninterface IAudioClipOpts {\n loop?: boolean;\n volume?: number;\n}\n\n/**\n * 音频素材，为创建、编辑音视频功能提供音频数据\n *\n * @example\n * new AudioClip((await fetch('')).body, {\n * loop: true,\n * }),\n */\nexport class AudioClip implements IClip {\n static ctx: AudioContext | null = null;\n\n ready: IClip['ready'];\n\n #meta = {\n // 微秒\n duration: 0,\n width: 0,\n height: 0,\n };\n\n /**\n * 音频元信息\n *\n * ⚠️ 注意，这里是转换后（标准化）的元信息，非原始音频元信息\n */\n get meta() {\n return {\n ...this.#meta,\n sampleRate: DEFAULT_AUDIO_CONF.sampleRate,\n chanCount: 2,\n };\n }\n\n // 使用类型断言来避免 ArrayBufferLike 和 ArrayBuffer 的类型兼容性问题\n #chan0Buf: Float32Array = new Float32Array();\n #chan1Buf: Float32Array = new Float32Array();\n /**\n * 获取音频素材完整的 PCM 数据\n */\n getPCMData(): Float32Array[] {\n return [this.#chan0Buf, this.#chan1Buf];\n }\n\n #opts;\n\n /**\n *\n * @param dataSource 音频文件流\n * @param opts 音频配置，控制音量、是否循环\n */\n constructor(\n dataSource: ReadableStream | Float32Array[],\n opts: IAudioClipOpts = {},\n ) {\n this.#opts = {\n loop: false,\n volume: 1,\n ...opts,\n };\n\n this.ready = this.#init(dataSource).then(() => ({\n // audio 没有宽高，无需绘制\n width: 0,\n height: 0,\n duration: opts.loop ? Infinity : this.#meta.duration,\n }));\n }\n\n async #init(\n dataSource: ReadableStream | Float32Array[],\n ): Promise {\n if (AudioClip.ctx == null) {\n AudioClip.ctx = new AudioContext({\n sampleRate: DEFAULT_AUDIO_CONF.sampleRate,\n });\n }\n\n const tStart = performance.now();\n const pcm =\n dataSource instanceof ReadableStream\n ? await parseStream2PCM(dataSource, AudioClip.ctx)\n : dataSource;\n\n Log.info('Audio clip decoded complete:', performance.now() - tStart);\n\n const volume = this.#opts.volume;\n if (volume !== 1) {\n for (const chan of pcm)\n for (let i = 0; i < chan.length; i += 1) chan[i] *= volume;\n }\n\n this.#meta.duration = (pcm[0].length / DEFAULT_AUDIO_CONF.sampleRate) * 1e6;\n\n this.#chan0Buf = pcm[0];\n // 单声道 转 立体声\n this.#chan1Buf = pcm[1] ?? this.#chan0Buf;\n\n Log.info(\n 'Audio clip convert to AudioData, time:',\n performance.now() - tStart,\n );\n }\n\n /**\n * 拦截 {@link AudioClip.tick} 方法返回的数据，用于对音频数据二次处理\n * @param time 调用 tick 的时间\n * @param tickRet tick 返回的数据\n *\n * @see [移除视频绿幕背景](https://webav-tech.github.io/WebAV/demo/3_2-chromakey-video)\n */\n tickInterceptor: >>(\n time: number,\n tickRet: T,\n ) => Promise = async (_, tickRet) => tickRet;\n\n // 微秒\n #ts = 0;\n #frameOffset = 0;\n /**\n * 返回上次与当前时刻差对应的音频 PCM 数据；\n *\n * 若差值超过 3s 或当前时间小于上次时间，则重置状态\n * @example\n * tick(0) // => []\n * tick(1e6) // => [leftChanPCM(1s), rightChanPCM(1s)]\n *\n */\n async tick(time: number): Promise<{\n audio: Float32Array[];\n state: 'success' | 'done';\n }> {\n if (!this.#opts.loop && time >= this.#meta.duration) {\n // 待观察：如果time跨度较大，返回done，理论上会丢失一些音频帧\n return await this.tickInterceptor(time, { audio: [], state: 'done' });\n }\n\n const deltaTime = time - this.#ts;\n\n // reset\n if (time < this.#ts || deltaTime > 3e6) {\n this.#ts = time;\n this.#frameOffset = Math.ceil(\n (this.#ts / 1e6) * DEFAULT_AUDIO_CONF.sampleRate,\n );\n return await this.tickInterceptor(time, {\n audio: [new Float32Array(0), new Float32Array(0)],\n state: 'success',\n });\n }\n\n this.#ts = time;\n const frameCnt = Math.ceil(\n (deltaTime / 1e6) * DEFAULT_AUDIO_CONF.sampleRate,\n );\n const endIdx = this.#frameOffset + frameCnt;\n const audio = this.#opts.loop\n ? [\n ringSliceFloat32Array(this.#chan0Buf, this.#frameOffset, endIdx),\n ringSliceFloat32Array(this.#chan1Buf, this.#frameOffset, endIdx),\n ]\n : [\n this.#chan0Buf.slice(this.#frameOffset, endIdx),\n this.#chan1Buf.slice(this.#frameOffset, endIdx),\n ];\n this.#frameOffset = endIdx;\n\n return await this.tickInterceptor(time, { audio, state: 'success' });\n }\n\n /**\n * 按指定时间切割，返回前后两个音频素材\n * @param time 时间，单位微秒\n */\n async split(time: number) {\n await this.ready;\n const frameCnt = Math.ceil((time / 1e6) * DEFAULT_AUDIO_CONF.sampleRate);\n const preSlice = new AudioClip(\n this.getPCMData().map((chan) => chan.slice(0, frameCnt)),\n this.#opts,\n );\n const postSlice = new AudioClip(\n this.getPCMData().map((chan) => chan.slice(frameCnt)),\n this.#opts,\n );\n return [preSlice, postSlice] as [this, this];\n }\n\n async clone() {\n await this.ready;\n const clip = new AudioClip(this.getPCMData(), this.#opts) as this;\n await clip.ready;\n return clip;\n }\n\n /**\n * 销毁实例，释放资源\n */\n destroy(): void {\n this.#chan0Buf = new Float32Array(0);\n this.#chan1Buf = new Float32Array(0);\n Log.info('---- audioclip destroy ----');\n }\n}\n\n/**\n * 拼接多个 AudioClip\n */\nexport async function concatAudioClip(\n clips: AudioClip[],\n opts?: IAudioClipOpts,\n) {\n const bufs: Float32Array[][] = [];\n for (const clip of clips) {\n await clip.ready;\n bufs.push(clip.getPCMData());\n }\n return new AudioClip(concatPCMFragments(bufs), opts);\n}\n\nasync function parseStream2PCM(\n stream: ReadableStream,\n ctx: AudioContext | OfflineAudioContext,\n): Promise {\n const buf = await new Response(stream).arrayBuffer();\n return extractPCM4AudioBuffer(await ctx.decodeAudioData(buf));\n}\n","import { autoReadStream } from '@webav/internal-utils';\nimport { IClip } from './iclip';\n\n/**\n * 包装实时音视频流，仅用于 [AVCanvas](../../av-canvas/classes/AVCanvas.html)\n *\n * ⚠️ 不可用于 {@link Combinator} ，因为后台合成视频的速度是快于物理时间的，实时流无法提供非实时的数据\n *\n * @example\n * const spr = new VisibleSprite(\n * new MediaStreamClip(\n * await navigator.mediaDevices.getUserMedia({ video: true, audio: true, }),\n * ),\n * );\n * await avCvs.addSprite(spr);\n */\nexport class MediaStreamClip implements IClip {\n static ctx: AudioContext | null = null;\n\n ready: IClip['ready'];\n\n #meta = {\n // 微秒\n duration: 0,\n width: 0,\n height: 0,\n };\n\n get meta() {\n return {\n ...this.#meta,\n };\n }\n\n #stopRenderCvs = () => {};\n\n /**\n * 实时流的音轨\n */\n readonly audioTrack: MediaStreamAudioTrack | null;\n\n #cvs: OffscreenCanvas | null = null;\n\n #ms: MediaStream;\n constructor(ms: MediaStream) {\n this.#ms = ms;\n this.audioTrack = ms.getAudioTracks()[0] ?? null;\n this.#meta.duration = Infinity;\n const videoTrack = ms.getVideoTracks()[0];\n if (videoTrack != null) {\n videoTrack.contentHint = 'motion';\n this.ready = new Promise((resolve) => {\n this.#stopRenderCvs = renderVideoTrackToCvs(videoTrack, (cvs) => {\n this.#meta.width = cvs.width;\n this.#meta.height = cvs.height;\n this.#cvs = cvs;\n resolve(this.meta);\n });\n });\n } else {\n this.ready = Promise.resolve(this.meta);\n }\n }\n\n async tick(): Promise<{\n video: ImageBitmap | null;\n audio: Float32Array[];\n state: 'success';\n }> {\n return {\n video: this.#cvs == null ? null : await createImageBitmap(this.#cvs),\n audio: [],\n state: 'success',\n };\n }\n\n async split() {\n return [await this.clone(), await this.clone()] as [this, this];\n }\n\n async clone() {\n return new MediaStreamClip(this.#ms.clone()) as this;\n }\n\n destroy(): void {\n this.#ms.getTracks().forEach((t) => t.stop());\n this.#stopRenderCvs();\n }\n}\n\nfunction renderVideoTrackToCvs(\n track: MediaStreamVideoTrack,\n onOffscreenCanvasReady: (cvs: OffscreenCanvas) => void,\n) {\n let emitFF = false;\n let cvsCtx: OffscreenCanvasRenderingContext2D;\n return autoReadStream(\n new MediaStreamTrackProcessor({\n track,\n }).readable,\n {\n onChunk: async (frame) => {\n if (!emitFF) {\n const { displayHeight, displayWidth } = frame;\n const width = displayWidth ?? 0;\n const height = displayHeight ?? 0;\n const cvs = new OffscreenCanvas(width, height);\n cvsCtx = cvs.getContext('2d')!;\n onOffscreenCanvasReady(cvs);\n emitFF = true;\n }\n cvsCtx.drawImage(frame, 0, 0);\n frame.close();\n },\n onDone: async () => {},\n },\n );\n}\n","import { IClip } from './iclip';\n\ninterface IEmbedSubtitlesOpts {\n color?: string;\n textBgColor?: string | null;\n type?: 'srt';\n fontFamily?: string;\n fontSize?: number;\n letterSpacing?: string | null;\n // 字幕偏离底部的距离\n bottomOffset?: number;\n strokeStyle?: string;\n lineWidth?: number | null;\n lineCap?: CanvasLineCap | null;\n lineJoin?: CanvasLineJoin | null;\n textShadow?: {\n offsetX: number;\n offsetY: number;\n blur: number;\n color: string;\n };\n videoWidth: number;\n videoHeight: number;\n fontWeight?: string | number;\n fontStyle?: 'normal' | 'italic';\n}\n\ndeclare global {\n interface OffscreenCanvasRenderingContext2D {\n letterSpacing: string;\n }\n}\n\ninterface SubtitleStruct {\n start: number;\n end: number;\n text: string;\n}\n\n/**\n * 嵌入式字幕，将字幕（目前仅支持 SRT 格式）嵌入视频画面中\n *\n * @example\n * const es = new EmbedSubtitlesClip(srtSubtitleStr, {\n * videoWidth: 1280,\n * videoHeight: 720,\n * fontFamily: 'Noto Sans SC',\n * color: 'white',\n * });\n */\nexport class EmbedSubtitlesClip implements IClip {\n ready: IClip['ready'];\n\n #subtitles: SubtitleStruct[] = [];\n\n #meta = {\n width: 0,\n height: 0,\n duration: 0,\n };\n\n get meta() {\n return { ...this.#meta };\n }\n\n #opts: Required = {\n color: '#FFF',\n textBgColor: null,\n type: 'srt',\n fontSize: 30,\n letterSpacing: null,\n bottomOffset: 30,\n fontFamily: 'Noto Sans SC',\n strokeStyle: '#000',\n lineWidth: null,\n lineCap: null,\n lineJoin: null,\n textShadow: {\n offsetX: 2,\n offsetY: 2,\n blur: 4,\n color: '#000',\n },\n videoWidth: 1280,\n videoHeight: 720,\n fontWeight: 'normal',\n fontStyle: 'normal',\n };\n\n #cvs: OffscreenCanvas;\n #ctx: OffscreenCanvasRenderingContext2D;\n\n #lastVF: VideoFrame | null = null;\n\n #lineHeight = 0;\n #linePadding = 0;\n\n constructor(content: string | SubtitleStruct[], opts: IEmbedSubtitlesOpts) {\n this.#subtitles = Array.isArray(content)\n ? content\n : parseSrt(content).map(({ start, end, text }) => ({\n start: start * 1e6,\n end: end * 1e6,\n text,\n }));\n if (this.#subtitles.length === 0) throw Error('No subtitles content');\n\n this.#opts = Object.assign(this.#opts, opts);\n // 如果需要绘制背景，则需要给文字添加边距\n this.#linePadding =\n opts.textBgColor == null ? 0 : (opts.fontSize ?? 50) * 0.2;\n\n const {\n fontSize,\n fontFamily,\n fontWeight,\n fontStyle,\n videoWidth,\n videoHeight,\n letterSpacing,\n } = this.#opts;\n this.#lineHeight = fontSize + this.#linePadding * 2;\n this.#cvs = new OffscreenCanvas(videoWidth, videoHeight);\n this.#ctx = this.#cvs.getContext('2d')!;\n this.#ctx.font = `${fontStyle} ${fontWeight} ${fontSize}px ${fontFamily}`;\n this.#ctx.textAlign = 'center';\n this.#ctx.textBaseline = 'top';\n this.#ctx.letterSpacing = letterSpacing ?? '0px';\n\n this.#meta = {\n width: videoWidth,\n height: videoHeight,\n duration: this.#subtitles.at(-1)?.end ?? 0,\n };\n // 字幕的宽高 由视频画面内容决定\n this.ready = Promise.resolve(this.meta);\n }\n\n #renderTxt(txt: string) {\n const lines = txt\n .split('\\n')\n .reverse()\n .map((t) => t.trim());\n\n const { width, height } = this.#cvs;\n\n const {\n color,\n fontSize,\n textBgColor,\n textShadow,\n strokeStyle,\n lineWidth,\n lineCap,\n lineJoin,\n bottomOffset,\n } = this.#opts;\n const ctx = this.#ctx;\n\n ctx.clearRect(0, 0, width, height);\n ctx.globalAlpha = 0.6;\n // 测试canvas背景\n // ctx.fillStyle = 'red'\n // ctx.fillRect(0, 0, this.#cvs.width, this.#cvs.height)\n\n let bottomDistance = bottomOffset;\n for (const lineStr of lines) {\n const txtMeas = ctx.measureText(lineStr);\n const centerX = width / 2;\n if (textBgColor != null) {\n ctx.shadowOffsetX = 0;\n ctx.shadowOffsetY = 0;\n ctx.shadowBlur = 0;\n // 字幕背景\n ctx.fillStyle = textBgColor;\n ctx.globalAlpha = 0.5;\n ctx.fillRect(\n centerX - txtMeas.actualBoundingBoxLeft - this.#linePadding,\n height - bottomDistance - this.#lineHeight,\n txtMeas.width + this.#linePadding * 2,\n this.#lineHeight,\n );\n } else {\n }\n\n ctx.shadowColor = textShadow.color;\n ctx.shadowOffsetX = textShadow.offsetX;\n ctx.shadowOffsetY = textShadow.offsetY;\n ctx.shadowBlur = textShadow.blur;\n\n ctx.globalAlpha = 1;\n\n if (strokeStyle != null) {\n ctx.lineWidth = lineWidth ?? fontSize / 6;\n if (lineCap != null) ctx.lineCap = lineCap;\n if (lineJoin != null) ctx.lineJoin = lineJoin;\n ctx.strokeStyle = strokeStyle;\n ctx.strokeText(\n lineStr,\n centerX,\n height - bottomDistance - this.#lineHeight + this.#linePadding,\n );\n }\n\n ctx.fillStyle = color;\n ctx.fillText(\n lineStr,\n centerX,\n height - bottomDistance - this.#lineHeight + this.#linePadding,\n );\n\n // 多行，底部偏移距离叠加\n bottomDistance += this.#lineHeight + fontSize * 0.2;\n }\n }\n\n /**\n * @see {@link IClip.tick}\n */\n async tick(time: number): Promise<{\n video?: VideoFrame;\n state: 'done' | 'success';\n }> {\n if (\n this.#lastVF != null &&\n time >= this.#lastVF.timestamp &&\n time <= this.#lastVF.timestamp + (this.#lastVF.duration ?? 0)\n ) {\n return { video: this.#lastVF.clone(), state: 'success' };\n }\n\n let i = 0;\n for (; i < this.#subtitles.length; i += 1) {\n if (time <= this.#subtitles[i].end) break;\n }\n\n const it = this.#subtitles[i] ?? this.#subtitles.at(-1);\n if (time > it.end) return { state: 'done' };\n if (time < it.start) {\n // 此时无字幕内容，清空画布\n this.#ctx.clearRect(0, 0, this.#cvs.width, this.#cvs.height);\n const vf = new VideoFrame(this.#cvs, {\n timestamp: time,\n // 直到下个字幕出现的时机\n duration: it.start - time,\n });\n this.#lastVF?.close();\n this.#lastVF = vf;\n\n return { video: vf.clone(), state: 'success' };\n }\n\n this.#renderTxt(it.text);\n\n const vf = new VideoFrame(this.#cvs, {\n timestamp: time,\n duration: it.end - time,\n });\n this.#lastVF?.close();\n this.#lastVF = vf;\n\n return { video: vf.clone(), state: 'success' };\n }\n\n /**\n * @see {@link IClip.split}\n */\n async split(time: number) {\n await this.ready;\n let hitIdx = -1;\n for (let i = 0; i < this.#subtitles.length; i++) {\n const sub = this.#subtitles[i];\n if (time > sub.start) continue;\n hitIdx = i;\n break;\n }\n if (hitIdx === -1) throw Error('Not found subtitle by time');\n const preSlice = this.#subtitles.slice(0, hitIdx).map((s) => ({ ...s }));\n let preLastIt = preSlice.at(-1);\n let postFirstIt = null;\n // 切割时间命中字幕区间，需要将当前字幕元素拆成前后两份\n if (preLastIt != null && preLastIt.end > time) {\n postFirstIt = {\n start: 0,\n end: preLastIt.end - time,\n text: preLastIt.text,\n };\n\n preLastIt.end = time;\n }\n const postSlice = this.#subtitles\n .slice(hitIdx)\n .map((s) => ({ ...s, start: s.start - time, end: s.end - time }));\n if (postFirstIt != null) postSlice.unshift(postFirstIt);\n return [\n new EmbedSubtitlesClip(preSlice, this.#opts),\n new EmbedSubtitlesClip(postSlice, this.#opts),\n ] as [this, this];\n }\n\n /**\n * @see {@link IClip.clone}\n */\n async clone() {\n return new EmbedSubtitlesClip(this.#subtitles.slice(0), this.#opts) as this;\n }\n\n /**\n * @see {@link IClip.destroy}\n */\n destroy() {\n this.#lastVF?.close();\n }\n}\n\n// SRT字幕格式 https://www.cnblogs.com/tocy/p/subtitle-format-srt.html\nfunction srtTimeToSeconds(time: string) {\n const match = time.match(/(\\d{2}):(\\d{2}):(\\d{2}),(\\d{3})/);\n if (match == null) throw Error(`time format error: ${time}`);\n\n const hours = Number(match[1]);\n const minutes = Number(match[2]);\n const seconds = Number(match[3]);\n const milliseconds = Number(match[4]);\n\n return hours * 60 * 60 + minutes * 60 + seconds + milliseconds / 1000;\n}\n\nfunction parseSrt(srt: string) {\n return (\n srt\n .split(/\\r|\\n/)\n .map((s) => s.trim())\n .filter((str) => str.length > 0)\n // 匹配时间戳标记行，匹配失败的为字幕内容\n .map((s) => ({\n lineStr: s,\n match: s.match(\n /(\\d{2}:\\d{2}:\\d{2},\\d{3}) --> (\\d{2}:\\d{2}:\\d{2},\\d{3})/,\n ),\n }))\n // 过滤掉时间上一行的数字标记\n .filter(\n ({ lineStr }, idx, source) =>\n !(/^\\d+$/.test(lineStr) && source[idx + 1]?.match != null),\n )\n // 按时间标记行聚合，拼接字幕内容到 text 字段\n .reduce(\n (acc, { lineStr, match }) => {\n if (match == null) {\n const last = acc.at(-1);\n if (last == null) return acc;\n\n last.text += last.text.length === 0 ? lineStr : `\\n${lineStr}`;\n } else {\n acc.push({\n start: srtTimeToSeconds(match[1]),\n end: srtTimeToSeconds(match[2]),\n text: '',\n });\n }\n\n return acc;\n },\n [] as Array<{\n start: number;\n end: number;\n text: string;\n }>,\n )\n );\n}\n","import mp4box, {\n MP4ArrayBuffer,\n MP4File,\n MP4Info,\n MP4Sample,\n} from '@webav/mp4box.js';\n\n/**\n * 将原始字节流转换成 MP4Sample 流\n */\nexport class SampleTransform {\n readable: ReadableStream<\n | {\n chunkType: 'ready';\n data: { info: MP4Info; file: MP4File };\n }\n | {\n chunkType: 'samples';\n data: { id: number; type: 'video' | 'audio'; samples: MP4Sample[] };\n }\n >;\n\n writable: WritableStream;\n\n #inputBufOffset = 0;\n\n constructor() {\n const file = mp4box.createFile();\n let streamCancelled = false;\n this.readable = new ReadableStream(\n {\n start: (ctrl) => {\n file.onReady = (info) => {\n const vTrackId = info.videoTracks[0]?.id;\n if (vTrackId != null)\n file.setExtractionOptions(vTrackId, 'video', { nbSamples: 100 });\n\n const aTrackId = info.audioTracks[0]?.id;\n if (aTrackId != null)\n file.setExtractionOptions(aTrackId, 'audio', { nbSamples: 100 });\n\n ctrl.enqueue({ chunkType: 'ready', data: { info, file } });\n file.start();\n };\n\n const releasedCnt: Record = {};\n file.onSamples = (id, type, samples) => {\n ctrl.enqueue({\n chunkType: 'samples',\n data: { id, type, samples: samples.map((s) => ({ ...s })) },\n });\n releasedCnt[id] = (releasedCnt[id] ?? 0) + samples.length;\n file.releaseUsedSamples(id, releasedCnt[id]);\n };\n\n file.onFlush = () => {\n ctrl.close();\n };\n },\n cancel: () => {\n file.stop();\n streamCancelled = true;\n },\n },\n {\n // 每条消息 100 个 samples\n highWaterMark: 50,\n },\n );\n\n this.writable = new WritableStream({\n write: async (ui8Arr) => {\n if (streamCancelled) {\n this.writable.abort();\n return;\n }\n\n const inputBuf = ui8Arr.buffer as MP4ArrayBuffer;\n inputBuf.fileStart = this.#inputBufOffset;\n this.#inputBufOffset += inputBuf.byteLength;\n file.appendBuffer(inputBuf);\n },\n close: () => {\n file.flush();\n file.stop();\n file.onFlush?.();\n },\n });\n }\n}\n","import mp4box, {\n MP4File,\n MP4Sample,\n SampleOpts,\n TrakBoxParser,\n} from '@webav/mp4box.js';\nimport { autoReadStream, file2stream, Log } from '@webav/internal-utils';\nimport {\n extractPCM4AudioData,\n extractPCM4AudioBuffer,\n mixinPCM,\n ringSliceFloat32Array,\n concatPCMFragments,\n} from '../av-utils';\nimport { DEFAULT_AUDIO_CONF } from '../clips';\nimport { SampleTransform } from './sample-transform';\nimport { extractFileConfig } from './mp4box-utils';\nimport { tmpfile, write } from 'opfs-tools';\n\nfunction fixMP4BoxFileDuration(\n inMP4File: MP4File,\n): () => Promise | null> {\n let sendedBoxIdx = 0;\n const boxes = inMP4File.boxes;\n const tracks: Array<{ track: TrakBoxParser; id: number }> = [];\n let totalDuration = 0;\n\n async function write2TmpFile() {\n const buf = box2Buf(boxes, sendedBoxIdx);\n sendedBoxIdx = boxes.length;\n // 释放引用，避免内存泄露\n // todo: use unsafeReleaseMP4BoxFile\n tracks.forEach(({ track, id }) => {\n const s = track.samples.at(-1);\n if (s != null)\n totalDuration = Math.max(totalDuration, s.cts + s.duration);\n\n inMP4File.releaseUsedSamples(id, track.samples.length);\n track.samples = [];\n });\n inMP4File.mdats = [];\n inMP4File.moofs = [];\n if (buf != null) await tmpFileWriter?.write(buf);\n }\n\n let moovPrevBoxes: typeof boxes = [];\n function moovBoxReady() {\n if (moovPrevBoxes.length > 0) return true;\n\n const moovIdx = boxes.findIndex((box) => box.type === 'moov');\n if (moovIdx === -1) return false;\n\n moovPrevBoxes = boxes.slice(0, moovIdx + 1);\n sendedBoxIdx = moovIdx + 1;\n\n if (tracks.length === 0) {\n for (let i = 1; true; i += 1) {\n const track = inMP4File.getTrackById(i);\n if (track == null) break;\n tracks.push({ track, id: i });\n }\n }\n\n return true;\n }\n\n let timerId = 0;\n // 把 moov 之外的 box 先写入临时文件，待更新 duration 之后再拼接临时文件\n const postFile = tmpfile();\n let tmpFileWriter: Awaited<\n ReturnType['createWriter']>\n > | null = null;\n\n const initPromise = (async () => {\n tmpFileWriter = await postFile.createWriter();\n\n timerId = self.setInterval(() => {\n if (!moovBoxReady()) return;\n write2TmpFile();\n }, 100);\n })();\n\n let stoped = false;\n return async () => {\n if (stoped) throw Error('File exported');\n stoped = true;\n\n await initPromise;\n clearInterval(timerId);\n\n if (!moovBoxReady() || tmpFileWriter == null) return null;\n inMP4File.flush();\n await write2TmpFile();\n await tmpFileWriter?.close();\n\n const moov = moovPrevBoxes.find((box) => box.type === 'moov') as\n | typeof inMP4File.moov\n | undefined;\n if (moov == null) return null;\n\n moov.mvhd.duration = totalDuration;\n\n const rsFile = tmpfile();\n const buf = box2Buf(moovPrevBoxes, 0)!;\n await write(rsFile, buf);\n await write(rsFile, postFile, { overwrite: false });\n\n return await rsFile.stream();\n };\n\n function box2Buf(source: typeof boxes, startIdx: number): Uint8Array | null {\n if (startIdx >= source.length) return null;\n\n const ds = new mp4box.DataStream();\n ds.endianness = mp4box.DataStream.BIG_ENDIAN;\n\n for (let i = startIdx; i < source.length; i++) {\n if (source[i] === null) continue;\n source[i].write(ds);\n delete source[i];\n }\n return new Uint8Array(ds.buffer);\n }\n}\n\n/**\n * EncodedAudioChunk | EncodedVideoChunk 转换为 MP4 addSample 需要的参数\n */\nfunction chunk2MP4SampleOpts(\n chunk: EncodedAudioChunk | EncodedVideoChunk,\n): SampleOpts & {\n data: ArrayBuffer;\n} {\n const buf = new ArrayBuffer(chunk.byteLength);\n chunk.copyTo(buf);\n const dts = chunk.timestamp;\n return {\n duration: chunk.duration ?? 0,\n dts,\n cts: dts,\n is_sync: chunk.type === 'key',\n data: buf,\n };\n}\n\n/**\n * 快速拼接多个mp4 文件流，要求所有 mp4 的属性一致，\n * 属性包括（不限于）：音视频编码格式、分辨率、采样率\n *\n * @param streams 一个包含 Uint8Array 的可读流数组。\n * @returns 返回一个 Promise，该 Promise 在解析时返回一个包含合并后的 MP4 数据的可读流。\n * @throws 如果无法从流生成文件，将抛出错误。\n *\n * @example\n * const streams = [stream1, stream2, stream3];\n * const resultStream = await fastConcatMP4(streams);\n */\nexport async function fastConcatMP4(\n streams: ReadableStream[],\n): Promise> {\n const outfile = mp4box.createFile();\n\n const dumpFile = fixMP4BoxFileDuration(outfile);\n await concatStreamsToMP4BoxFile(streams, outfile);\n const outStream = await dumpFile();\n if (outStream == null) throw Error('Can not generate file from streams');\n return outStream;\n}\n\nasync function concatStreamsToMP4BoxFile(\n streams: ReadableStream[],\n outfile: MP4File,\n) {\n let vTrackId = 0;\n let vDTS = 0;\n let vCTS = 0;\n let aTrackId = 0;\n let aDTS = 0;\n let aCTS = 0;\n // ts bug, 不能正确识别类型\n let lastVSamp: any = null;\n let lastASamp: any = null;\n for (const stream of streams) {\n await new Promise(async (resolve) => {\n autoReadStream(stream.pipeThrough(new SampleTransform()), {\n onDone: resolve,\n onChunk: async ({ chunkType, data }) => {\n if (chunkType === 'ready') {\n const { videoTrackConf, audioTrackConf } = extractFileConfig(\n data.file,\n data.info,\n );\n if (vTrackId === 0 && videoTrackConf != null) {\n vTrackId = outfile.addTrack(videoTrackConf);\n }\n if (aTrackId === 0 && audioTrackConf != null) {\n aTrackId = outfile.addTrack(audioTrackConf);\n }\n } else if (chunkType === 'samples') {\n const { type, samples } = data;\n const trackId = type === 'video' ? vTrackId : aTrackId;\n const offsetDTS = type === 'video' ? vDTS : aDTS;\n const offsetCTS = type === 'video' ? vCTS : aCTS;\n\n samples.forEach((s) => {\n outfile.addSample(trackId, s.data, {\n duration: s.duration,\n dts: s.dts + offsetDTS,\n cts: s.cts + offsetCTS,\n is_sync: s.is_sync,\n });\n });\n\n const lastSamp = samples.at(-1);\n if (lastSamp == null) return;\n if (type === 'video') {\n lastVSamp = lastSamp;\n } else if (type === 'audio') {\n lastASamp = lastSamp;\n }\n }\n },\n });\n });\n if (lastVSamp != null) {\n vDTS += lastVSamp.dts;\n vCTS += lastVSamp.cts;\n }\n if (lastASamp != null) {\n aDTS += lastASamp.dts;\n aCTS += lastASamp.cts;\n }\n }\n}\n\n/**\n * 为 WebAV 生成的 fmp4 文件设置正确的时长值\n */\nexport async function fixFMP4Duration(\n stream: ReadableStream,\n): Promise> {\n return await fastConcatMP4([stream]);\n}\n\n/**\n * 创建 MP4 音频样本解码器。\n * @param adConf - 音频解码器配置参数 {@link AudioDecoderConfig}。\n * @returns 返回一个对象，包含 `decode` 和 `close` 方法。\n * - `decode` 方法用于解码 MP4 音频样本，返回解码后的音频数据数组。\n * - `close` 方法用于关闭音频解码器。\n */\nfunction createMP4AudioSampleDecoder(\n adConf: Parameters[0],\n) {\n let cacheAD: AudioData[] = [];\n const adDecoder = new AudioDecoder({\n output: (ad) => {\n cacheAD.push(ad);\n },\n error: Log.error,\n });\n adDecoder.configure(adConf);\n\n return {\n decode: async (ss: MP4Sample[]) => {\n ss.forEach((s) => {\n adDecoder.decode(\n new EncodedAudioChunk({\n type: s.is_sync ? 'key' : 'delta',\n timestamp: (1e6 * s.cts) / s.timescale,\n duration: (1e6 * s.duration) / s.timescale,\n data: s.data,\n }),\n );\n });\n\n await adDecoder.flush();\n\n const rs = cacheAD;\n cacheAD = [];\n\n return rs;\n },\n close: () => {\n adDecoder.close();\n },\n };\n}\n\n// 音频编码与解码API有很大区别，\n// 是因为编码中途调用 AudioEncoder.flush ，会导致声音听起来卡顿\nfunction createMP4AudioSampleEncoder(\n aeConf: Parameters[0],\n onOutput: (s: ReturnType) => void,\n) {\n const encoderConf = {\n codec: aeConf.codec,\n sampleRate: aeConf.sampleRate,\n numberOfChannels: aeConf.numberOfChannels,\n } as const;\n\n const adEncoder = new AudioEncoder({\n output: (chunk) => {\n onOutput(chunk2MP4SampleOpts(chunk));\n },\n error: (err) => {\n Log.error('AudioEncoder error:', err, ', config:', encoderConf);\n },\n });\n\n adEncoder.configure(encoderConf);\n\n // 保留一个音频数据，用于最后做声音淡出\n let lastData: { data: Float32Array; ts: number } | null = null;\n\n function createAD(data: Float32Array, ts: number) {\n return new AudioData({\n timestamp: ts,\n numberOfChannels: aeConf.numberOfChannels,\n numberOfFrames: data.length / aeConf.numberOfChannels,\n sampleRate: aeConf.sampleRate,\n format: 'f32-planar',\n data,\n });\n }\n return {\n encode: async (data: Float32Array, ts: number) => {\n if (lastData != null) {\n adEncoder.encode(createAD(lastData.data, lastData.ts));\n }\n lastData = { data, ts };\n },\n stop: async () => {\n if (lastData != null) {\n // 副作用修改数据\n audioFade(lastData.data, aeConf.numberOfChannels, aeConf.sampleRate);\n adEncoder.encode(createAD(lastData.data, lastData.ts));\n lastData = null;\n }\n await adEncoder.flush();\n adEncoder.close();\n },\n };\n}\n\n/**\n * 音频线性淡出，避免 POP 声\n * 副作用调整音量值\n */\nfunction audioFade(pcmData: Float32Array, chanCnt: number, sampleRate: number) {\n const dataLen = pcmData.length - 1;\n // 避免超出边界，最长 500ms 的淡出时间\n const fadeLen = Math.min(sampleRate / 2, dataLen);\n for (let i = 0; i < fadeLen; i++) {\n for (let j = 1; j <= chanCnt; j++) {\n // 从尾部开始，调整每个声道音量值\n pcmData[Math.floor(dataLen / j) - i] *= i / fadeLen;\n }\n }\n}\n\n/**\n * 视频配音；混合 MP4 与音频文件，仅重编码音频，视频轨道不变\n * @param mp4Stream - MP4 流\n * @param audio - 音频信息\n * @param audio.stream - 音频数据流\n * @param audio.volume - 音频音量\n * @param audio.loop - 音频时长小于视频时，是否循环使用音频流\n * @returns 输出混合后的音频流\n */\nexport function mixinMP4AndAudio(\n mp4Stream: ReadableStream,\n audio: {\n stream: ReadableStream;\n volume: number;\n loop: boolean;\n },\n) {\n Log.info('mixinMP4AndAudio, opts:', {\n volume: audio.volume,\n loop: audio.loop,\n });\n\n const outfile = mp4box.createFile();\n const { stream: outStream, stop: stopOut } = file2stream(outfile, 500);\n\n let audioSampleDecoder: ReturnType<\n typeof createMP4AudioSampleDecoder\n > | null = null;\n\n let audioSampleEncoder: ReturnType<\n typeof createMP4AudioSampleEncoder\n > | null = null;\n\n let inputAudioPCM: Float32Array[] = [];\n\n let vTrackId = 0;\n let aTrackId = 0;\n let audioOffset = 0;\n let mp4HasAudio = true;\n let sampleRate = DEFAULT_AUDIO_CONF.sampleRate as number;\n autoReadStream(mp4Stream.pipeThrough(new SampleTransform()), {\n onDone: async () => {\n await audioSampleEncoder?.stop();\n audioSampleDecoder?.close();\n stopOut();\n },\n onChunk: async ({ chunkType, data }) => {\n if (chunkType === 'ready') {\n const { videoTrackConf, audioTrackConf, audioDecoderConf } =\n extractFileConfig(data.file, data.info);\n if (vTrackId === 0 && videoTrackConf != null) {\n vTrackId = outfile.addTrack(videoTrackConf);\n }\n\n const safeAudioTrackConf = audioTrackConf ?? {\n timescale: 1e6,\n samplerate: sampleRate,\n channel_count: DEFAULT_AUDIO_CONF.channelCount,\n hdlr: 'soun',\n name: 'SoundHandler',\n type: 'mp4a',\n };\n if (aTrackId === 0) {\n aTrackId = outfile.addTrack(safeAudioTrackConf);\n sampleRate = audioTrackConf?.samplerate ?? sampleRate;\n mp4HasAudio = audioTrackConf == null ? false : true;\n }\n const audioCtx = new AudioContext({ sampleRate });\n inputAudioPCM = extractPCM4AudioBuffer(\n await audioCtx.decodeAudioData(\n await new Response(audio.stream).arrayBuffer(),\n ),\n );\n\n if (audioDecoderConf != null) {\n audioSampleDecoder = createMP4AudioSampleDecoder(audioDecoderConf);\n }\n audioSampleEncoder = createMP4AudioSampleEncoder(\n audioDecoderConf ?? {\n codec:\n safeAudioTrackConf.type === 'mp4a'\n ? DEFAULT_AUDIO_CONF.codec\n : safeAudioTrackConf.type,\n numberOfChannels: safeAudioTrackConf.channel_count,\n sampleRate: safeAudioTrackConf.samplerate,\n },\n (s) => outfile.addSample(aTrackId, s.data, s),\n );\n } else if (chunkType === 'samples') {\n const { id, type, samples } = data;\n if (type === 'video') {\n samples.forEach((s) => outfile.addSample(id, s.data, s));\n\n if (!mp4HasAudio) await addInputAudio2Track(samples);\n return;\n }\n\n if (type === 'audio') await mixinAudioSampleAndInputPCM(samples);\n }\n },\n });\n\n function getInputAudioSlice(len: number) {\n const rs = inputAudioPCM.map((chanBuf) =>\n audio.loop\n ? ringSliceFloat32Array(chanBuf, audioOffset, audioOffset + len)\n : chanBuf.slice(audioOffset, audioOffset + len),\n );\n audioOffset += len;\n\n if (audio.volume !== 1) {\n for (const buf of rs)\n for (let i = 0; i < buf.length; i++) buf[i] *= audio.volume;\n }\n\n return rs;\n }\n\n async function addInputAudio2Track(vdieoSamples: MP4Sample[]) {\n const firstSamp = vdieoSamples[0];\n const lastSamp = vdieoSamples[vdieoSamples.length - 1];\n const pcmLength = Math.floor(\n ((lastSamp.cts + lastSamp.duration - firstSamp.cts) /\n lastSamp.timescale) *\n sampleRate,\n );\n const audioDataBuf = mixinPCM([getInputAudioSlice(pcmLength)]);\n if (audioDataBuf.length === 0) return;\n audioSampleEncoder?.encode(\n audioDataBuf,\n (firstSamp.cts / firstSamp.timescale) * 1e6,\n );\n }\n\n async function mixinAudioSampleAndInputPCM(samples: MP4Sample[]) {\n if (audioSampleDecoder == null) return;\n\n // 1. 先解码mp4音频\n // [[chan0, chan1], [chan0, chan1]...]\n const pcmFragments = (await audioSampleDecoder.decode(samples)).map(\n extractPCM4AudioData,\n );\n // [chan0, chan1]\n const mp4AudioPCM = concatPCMFragments(pcmFragments);\n const inputAudioPCM = getInputAudioSlice(mp4AudioPCM[0].length);\n const firstSamp = samples[0];\n\n // 3. 重编码音频\n audioSampleEncoder?.encode(\n // 2. 混合输入的音频\n mixinPCM([mp4AudioPCM, inputAudioPCM]),\n (firstSamp.cts / firstSamp.timescale) * 1e6,\n );\n }\n\n return outStream;\n}\n","// 改编自 https://jameshfisher.com/2020/08/11/production-ready-green-screen-in-the-browser/\nconst vertexShader = `#version 300 es\n layout (location = 0) in vec4 a_position;\n layout (location = 1) in vec2 a_texCoord;\n out vec2 v_texCoord;\n void main () {\n gl_Position = a_position;\n v_texCoord = a_texCoord;\n }\n`;\n\nconst fragmentShader = `#version 300 es\nprecision mediump float;\nout vec4 FragColor;\nin vec2 v_texCoord;\n\nuniform sampler2D frameTexture;\nuniform vec3 keyColor;\n\n// 色度的相似度计算\nuniform float similarity;\n// 透明度的平滑度计算\nuniform float smoothness;\n// 降低绿幕饱和度，提高抠图准确度\nuniform float spill;\n\nvec2 RGBtoUV(vec3 rgb) {\n return vec2(\n rgb.r * -0.169 + rgb.g * -0.331 + rgb.b * 0.5 + 0.5,\n rgb.r * 0.5 + rgb.g * -0.419 + rgb.b * -0.081 + 0.5\n );\n}\n\nvoid main() {\n // 获取当前像素的rgba值\n vec4 rgba = texture(frameTexture, v_texCoord);\n // 计算当前像素与绿幕像素的色度差值\n vec2 chromaVec = RGBtoUV(rgba.rgb) - RGBtoUV(keyColor);\n // 计算当前像素与绿幕像素的色度距离（向量长度）, 越相像则色度距离越小\n float chromaDist = sqrt(dot(chromaVec, chromaVec));\n // 设置了一个相似度阈值，baseMask为负，则表明是绿幕，为正则表明不是绿幕\n float baseMask = chromaDist - similarity;\n // 如果baseMask为负数，fullMask等于0；baseMask为正数，越大，则透明度越低\n float fullMask = pow(clamp(baseMask / smoothness, 0., 1.), 1.5);\n rgba.a = fullMask; // 设置透明度\n // 如果baseMask为负数，spillVal等于0；baseMask为整数，越小，饱和度越低\n float spillVal = pow(clamp(baseMask / spill, 0., 1.), 1.5);\n float desat = clamp(rgba.r * 0.2126 + rgba.g * 0.7152 + rgba.b * 0.0722, 0., 1.); // 计算当前像素的灰度值\n rgba.rgb = mix(vec3(desat, desat, desat), rgba.rgb, spillVal);\n FragColor = rgba;\n}\n`;\n\nconst POINT_POS = [-1, 1, -1, -1, 1, -1, 1, -1, 1, 1, -1, 1];\nconst TEX_COORD_POS = [0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1];\n\n// 初始化着色器程序，让 WebGL 知道如何绘制我们的数据\nfunction initShaderProgram(\n gl: WebGLRenderingContext,\n vsSource: string,\n fsSource: string,\n) {\n const vertexShader = loadShader(gl, gl.VERTEX_SHADER, vsSource)!;\n const fragmentShader = loadShader(gl, gl.FRAGMENT_SHADER, fsSource)!;\n\n // 创建着色器程序\n const shaderProgram = gl.createProgram()!;\n gl.attachShader(shaderProgram, vertexShader);\n gl.attachShader(shaderProgram, fragmentShader);\n gl.linkProgram(shaderProgram);\n\n if (!gl.getProgramParameter(shaderProgram, gl.LINK_STATUS)) {\n throw Error(\n gl.getProgramInfoLog(shaderProgram) ??\n 'Unable to initialize the shader program',\n );\n }\n\n return shaderProgram;\n}\n\n// 创建指定类型的着色器，上传 source 源码并编译\nfunction loadShader(gl: WebGLRenderingContext, type: number, source: string) {\n const shader = gl.createShader(type)!;\n\n // Send the source to the shader object\n gl.shaderSource(shader, source);\n\n // Compile the shader program\n gl.compileShader(shader);\n\n // See if it compiled successfully\n if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {\n const errMsg = gl.getShaderInfoLog(shader);\n gl.deleteShader(shader);\n throw Error(errMsg ?? 'An error occurred compiling the shaders');\n }\n\n return shader;\n}\n\nfunction updateTexture(\n gl: WebGLRenderingContext,\n img: TImgSource,\n texture: WebGLTexture,\n) {\n gl.bindTexture(gl.TEXTURE_2D, texture);\n gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, img);\n gl.drawArrays(gl.TRIANGLES, 0, 6);\n}\n\nfunction initTexture(gl: WebGLRenderingContext) {\n const texture = gl.createTexture();\n if (texture == null) throw Error('Create WebGL texture error');\n gl.bindTexture(gl.TEXTURE_2D, texture);\n\n // put a single pixel in the texture so we can use it immediately.\n const level = 0;\n const internalFormat = gl.RGBA;\n const width = 1;\n const height = 1;\n const border = 0;\n const srcFormat = gl.RGBA;\n const srcType = gl.UNSIGNED_BYTE;\n const pixel = new Uint8Array([0, 0, 255, 255]); // opaque blue\n gl.texImage2D(\n gl.TEXTURE_2D,\n level,\n internalFormat,\n width,\n height,\n border,\n srcFormat,\n srcType,\n pixel,\n );\n\n gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);\n gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);\n gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);\n gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);\n\n return texture;\n}\n\ninterface IChromakeyOpts {\n keyColor: [number, number, number];\n similarity: number;\n smoothness: number;\n spill: number;\n}\n\nfunction initCvs(\n opts: {\n width: number;\n height: number;\n } & IChromakeyOpts,\n) {\n const cvs =\n 'document' in globalThis\n ? globalThis.document.createElement('canvas')\n : new OffscreenCanvas(opts.width, opts.height);\n cvs.width = opts.width;\n cvs.height = opts.height;\n\n const gl = cvs.getContext('webgl2', {\n premultipliedAlpha: false,\n alpha: true,\n }) as WebGL2RenderingContext | null;\n\n if (gl == null) throw Error('Cant create gl context');\n\n const shaderProgram = initShaderProgram(gl, vertexShader, fragmentShader);\n gl.useProgram(shaderProgram);\n\n gl.uniform3fv(\n gl.getUniformLocation(shaderProgram, 'keyColor'),\n opts.keyColor.map((v) => v / 255),\n );\n gl.uniform1f(\n gl.getUniformLocation(shaderProgram, 'similarity'),\n opts.similarity,\n );\n gl.uniform1f(\n gl.getUniformLocation(shaderProgram, 'smoothness'),\n opts.smoothness,\n );\n gl.uniform1f(gl.getUniformLocation(shaderProgram, 'spill'), opts.spill);\n\n const posBuffer = gl.createBuffer();\n gl.bindBuffer(gl.ARRAY_BUFFER, posBuffer);\n gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(POINT_POS), gl.STATIC_DRAW);\n const a_position = gl.getAttribLocation(shaderProgram, 'a_position');\n gl.vertexAttribPointer(\n a_position,\n 2,\n gl.FLOAT,\n false,\n Float32Array.BYTES_PER_ELEMENT * 2,\n 0,\n );\n gl.enableVertexAttribArray(a_position);\n\n const texCoordBuffer = gl.createBuffer();\n gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);\n gl.bufferData(\n gl.ARRAY_BUFFER,\n new Float32Array(TEX_COORD_POS),\n gl.STATIC_DRAW,\n );\n const a_texCoord = gl.getAttribLocation(shaderProgram, 'a_texCoord');\n gl.vertexAttribPointer(\n a_texCoord,\n 2,\n gl.FLOAT,\n false,\n Float32Array.BYTES_PER_ELEMENT * 2,\n 0,\n );\n gl.enableVertexAttribArray(a_texCoord);\n\n gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, 1);\n\n return { cvs, gl };\n}\n\ntype TImgSource =\n | HTMLVideoElement\n | HTMLCanvasElement\n | HTMLImageElement\n | ImageBitmap\n | OffscreenCanvas\n | VideoFrame;\n\nfunction getSourceWH(imgSource: TImgSource) {\n return imgSource instanceof VideoFrame\n ? { width: imgSource.codedWidth, height: imgSource.codedHeight }\n : { width: imgSource.width, height: imgSource.height };\n}\n\nfunction getKeyColor(imgSource: TImgSource) {\n const cvs = new OffscreenCanvas(1, 1);\n const ctx = cvs.getContext('2d')!;\n ctx.drawImage(imgSource, 0, 0);\n const {\n data: [r, g, b],\n } = ctx.getImageData(0, 0, 1, 1);\n return [r, g, b] as [number, number, number];\n}\n\n/**\n * 绿幕抠图\n * keyColor 需要扣除的背景色，若不传则取第一个像素点\n * similarity 背景色相似度阈值，过小可能保留背景色，过大可能扣掉更多非背景像素点\n * smoothness 平滑度；过小可能出现锯齿，过大导致整体变透明\n * spill 饱和度；过小可能保留绿色混合，过大导致图片变灰度\n * @param opts: {\n * keyColor?: [r, g, b]\n * similarity: number\n * smoothness: number\n * spill: number\n * }\n */\nexport const createChromakey = (\n opts: Omit & {\n keyColor?: [number, number, number];\n },\n) => {\n let cvs: HTMLCanvasElement | OffscreenCanvas | null = null;\n let gl: WebGLRenderingContext | null = null;\n let keyC = opts.keyColor;\n let texture: WebGLTexture | null = null;\n\n return async (imgSource: TImgSource) => {\n if (cvs == null || gl == null || texture == null) {\n if (keyC == null) keyC = getKeyColor(imgSource);\n ({ cvs, gl } = initCvs({\n ...getSourceWH(imgSource),\n keyColor: keyC,\n ...opts,\n }));\n texture = initTexture(gl);\n }\n\n updateTexture(gl, imgSource, texture);\n\n if (\n globalThis.VideoFrame != null &&\n imgSource instanceof globalThis.VideoFrame\n ) {\n const rs = new VideoFrame(cvs, {\n alpha: 'keep',\n timestamp: imgSource.timestamp,\n duration: imgSource.duration ?? undefined,\n });\n imgSource.close();\n return rs;\n }\n\n return createImageBitmap(cvs, {\n imageOrientation: imgSource instanceof ImageBitmap ? 'flipY' : 'none',\n });\n };\n};\n","import { EventTool } from '@webav/internal-utils';\n\ninterface IPoint {\n x: number;\n y: number;\n}\n\nexport interface IRectBaseProps {\n x: number;\n y: number;\n w: number;\n h: number;\n angle: number;\n}\n\n/**\n * 用于记录素材在视频或画布中的空间属性：位置、大小、旋转\n *\n * 并提供控制点位置，支持用户在画布中缩放、旋转素材\n *\n * 一般由内部 WebAV SDK 内部创建维护\n *\n * @see {@link Combinator}, {@link OffscreenSprite}\n * @see [AVCanvas](../../av-canvas/classes/AVCanvas.html), {@link VisibleSprite}\n *\n * @see [视频剪辑](https://webav-tech.github.io/WebAV/demo/6_4-video-editor)\n */\nexport class Rect implements IRectBaseProps {\n #evtTool = new EventTool<{\n propsChange: (props: Partial) => void;\n }>();\n /**\n * 监听属性变更事件\n * @example\n * rect.on('propsChange', (changedProps) => {})\n */\n on = this.#evtTool.on;\n\n #x = 0;\n /**\n * x 坐标\n */\n get x() {\n return this.#x;\n }\n set x(v) {\n this.#setBaseProps('x', v);\n }\n #y = 0;\n get y() {\n return this.#y;\n }\n /**\n * y 坐标\n */\n set y(v) {\n this.#setBaseProps('y', v);\n }\n #w = 0;\n /**\n * 宽\n */\n get w() {\n return this.#w;\n }\n set w(v) {\n this.#setBaseProps('w', v);\n }\n #h = 0;\n /**\n * 高\n */\n get h() {\n return this.#h;\n }\n set h(v) {\n this.#setBaseProps('h', v);\n }\n #angle = 0;\n /**\n * 旋转角度\n * @see [MDN Canvas rotate](https://developer.mozilla.org/docs/Web/API/CanvasRenderingContext2D/rotate)\n */\n get angle() {\n return this.#angle;\n }\n set angle(v) {\n this.#setBaseProps('angle', v);\n }\n\n #setBaseProps(prop: keyof IRectBaseProps, v: number) {\n const changed = this[prop] !== v;\n switch (prop) {\n case 'x':\n this.#x = v;\n break;\n case 'y':\n this.#y = v;\n break;\n case 'w':\n this.#w = v;\n break;\n case 'h':\n this.#h = v;\n break;\n case 'angle':\n this.#angle = v;\n break;\n }\n if (changed) this.#evtTool.emit('propsChange', { [prop]: v });\n }\n\n /**\n * 如果当前实例是 Rect 控制点之一，`master` 将指向该 Rect\n *\n * 控制点的坐标是相对于它的 `master` 定位\n */\n #master: Rect | null = null;\n\n constructor(\n x?: number,\n y?: number,\n w?: number,\n h?: number,\n master?: Rect | null,\n ) {\n this.x = x ?? 0;\n this.y = y ?? 0;\n this.w = w ?? 0;\n this.h = h ?? 0;\n this.#master = master ?? null;\n }\n\n /**\n * 根据坐标、宽高计算出来的矩形中心点\n */\n get center(): IPoint {\n const { x, y, w, h } = this;\n return { x: x + w / 2, y: y + h / 2 };\n }\n\n /**\n * 是否保持固定宽高比例，禁止变形缩放\n *\n * 值为 true 时，将缺少上下左右四个控制点\n */\n fixedAspectRatio = false;\n\n /**\n * 是否固定中心点进行缩放\n * 值为 true 时，固定中心点不变进行缩放\n * 值为 false 时，固定对角点不变进行缩放\n */\n fixedScaleCenter = false;\n\n clone(): Rect {\n const { x, y, w, h } = this;\n const rect = new Rect(x, y, w, h, this.#master);\n rect.angle = this.angle;\n rect.fixedAspectRatio = this.fixedAspectRatio;\n rect.fixedScaleCenter = this.fixedScaleCenter;\n return rect;\n }\n\n /**\n * 检测目标坐标是否命中当前实例\n * @param tx 目标点 x 坐标\n * @param ty 目标点 y 坐标\n */\n checkHit(tx: number, ty: number): boolean {\n let { angle, center, x, y, w, h } = this;\n // ctrls 的中心点、旋转角度都取自于 master （sprite）\n const cnt = this.#master?.center ?? center;\n const agl = this.#master?.angle ?? angle;\n // ctrl 初始化时其坐标就是相对于 master 的，参见 get ctrls()\n // 所以此处不用转换\n if (this.#master == null) {\n x = x - cnt.x;\n y = y - cnt.y;\n }\n // 鼠标点击坐标映射成以中点为原点的坐标\n const tOX = tx - cnt.x;\n const tOY = ty - cnt.y;\n // 如果有旋转，映射成相对 sprite 原点，旋转前的坐标\n let mx = tOX;\n let my = tOY;\n if (agl !== 0) {\n // 推导公式 https://github.com/hughfenghen/hughfenghen.github.io/issues/96\n mx = tOX * Math.cos(agl) + tOY * Math.sin(agl);\n my = tOY * Math.cos(agl) - tOX * Math.sin(agl);\n }\n\n if (mx < x || mx > x + w || my < y || my > y + h) return false;\n\n return true;\n }\n}\n","import { EventTool } from '@webav/internal-utils';\nimport { IRectBaseProps, Rect } from './rect';\n\ninterface IAnimationOpts {\n duration: number;\n delay?: number;\n iterCount?: number;\n}\n\ntype TAnimateProps = IRectBaseProps & { opacity: number };\n\nexport type TAnimationKeyFrame = Array<[number, Partial]>;\n\ntype TKeyFrameOpts = Partial<\n Record<`${number}%` | 'from' | 'to', Partial>\n>;\n\n/**\n * Sprite 基类\n *\n * @see {@link OffscreenSprite}\n * @see {@link VisibleSprite}\n */\nexport abstract class BaseSprite {\n /**\n * 控制素材在视频中的空间属性（坐标、旋转、缩放）\n */\n rect = new Rect();\n\n /**\n * 控制素材在的时间偏移、时长、播放速率，常用于剪辑场景时间轴（轨道）模块\n * duration 不能大于引用 {@link IClip} 的时长，单位 微秒\n *\n * playbackRate 控制当前素材的播放速率，1 表示正常播放；\n * **注意**\n * 1. 设置 playbackRate 时需要主动修正 duration\n * 2. 音频使用最简单的插值算法来改变速率，所以改变速率后音调会产生变化，自定义算法请使用 {@link MP4Clip.tickInterceptor} 配合实现\n *\n */\n #time = {\n offset: 0,\n duration: 0,\n playbackRate: 1,\n };\n get time(): { offset: number; duration: number; playbackRate: number } {\n return this.#time;\n }\n set time(v: { offset: number; duration: number; playbackRate?: number }) {\n Object.assign(this.#time, v);\n }\n\n #evtTool = new EventTool<{\n propsChange: (\n value: Partial<{ rect: Partial; zIndex: number }>,\n ) => void;\n }>();\n /**\n * 监听属性变更事件\n * @example\n * sprite.on('propsChange', (changedProps) => {})\n */\n on = this.#evtTool.on;\n\n #zIndex = 0;\n get zIndex(): number {\n return this.#zIndex;\n }\n\n /**\n * 控制素材间的层级关系，zIndex 值较小的素材会被遮挡\n */\n set zIndex(v: number) {\n const changed = this.#zIndex !== v;\n this.#zIndex = v;\n if (changed) this.#evtTool.emit('propsChange', { zIndex: v });\n }\n\n /**\n * 不透明度\n */\n opacity = 1;\n\n /**\n * 水平或垂直方向翻转素材\n */\n flip: 'horizontal' | 'vertical' | null = null;\n\n #animatKeyFrame: TAnimationKeyFrame | null = null;\n\n #animatOpts: Required | null = null;\n\n /**\n * @see {@link IClip.ready}\n */\n ready = Promise.resolve();\n\n constructor() {\n this.rect.on('propsChange', (props) => {\n this.#evtTool.emit('propsChange', { rect: props });\n });\n }\n\n protected _render(\n ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,\n ): void {\n const {\n rect: { center, angle },\n } = this;\n ctx.setTransform(\n // 水平 缩放、倾斜\n this.flip === 'horizontal' ? -1 : 1,\n 0,\n // 垂直 倾斜、缩放\n 0,\n this.flip === 'vertical' ? -1 : 1,\n // 坐标原点偏移 x y\n center.x,\n center.y,\n );\n // 任意方向翻转，旋转角度转为负值，才能与控制点同步\n ctx.rotate((this.flip == null ? 1 : -1) * angle);\n\n ctx.globalAlpha = this.opacity;\n }\n\n /**\n * 给素材添加动画，使用方法参考 css animation\n *\n * @example\n * sprite.setAnimation(\n * {\n * '0%': { x: 0, y: 0 },\n * '25%': { x: 1200, y: 680 },\n * '50%': { x: 1200, y: 0 },\n * '75%': { x: 0, y: 680 },\n * '100%': { x: 0, y: 0 },\n * },\n * { duration: 4e6, iterCount: 1 },\n * );\n *\n * @see [视频水印动画](https://webav-tech.github.io/WebAV/demo/2_1-concat-video)\n */\n setAnimation(keyFrame: TKeyFrameOpts, opts: IAnimationOpts): void {\n this.#animatKeyFrame = Object.entries(keyFrame).map(([k, val]) => {\n const numK = { from: 0, to: 100 }[k] ?? Number(k.slice(0, -1));\n if (isNaN(numK) || numK > 100 || numK < 0) {\n throw Error('keyFrame must between 0~100');\n }\n return [numK / 100, val];\n }) as TAnimationKeyFrame;\n this.#animatOpts = Object.assign({}, this.#animatOpts, {\n duration: opts.duration,\n delay: opts.delay ?? 0,\n iterCount: opts.iterCount ?? Infinity,\n });\n }\n\n /**\n * 如果当前 sprite 已被设置动画，将 sprite 的动画属性设定到指定时间的状态\n */\n animate(time: number): void {\n if (\n this.#animatKeyFrame == null ||\n this.#animatOpts == null ||\n time < this.#animatOpts.delay\n )\n return;\n const updateProps = linearTimeFn(\n time,\n this.#animatKeyFrame,\n this.#animatOpts,\n );\n for (const k in updateProps) {\n switch (k) {\n case 'opacity':\n this.opacity = updateProps[k] as number;\n break;\n case 'x':\n case 'y':\n case 'w':\n case 'h':\n case 'angle':\n this.rect[k] = updateProps[k] as number;\n break;\n }\n }\n }\n\n /**\n * 将当前 sprite 的属性赋值到目标\n *\n * 用于 clone，或 {@link VisibleSprite} 与 {@link OffscreenSprite} 实例间的类型转换\n */\n copyStateTo(target: T) {\n target.#animatKeyFrame = this.#animatKeyFrame;\n target.#animatOpts = this.#animatOpts;\n target.zIndex = this.zIndex;\n target.opacity = this.opacity;\n target.flip = this.flip;\n target.rect = this.rect.clone();\n target.time = { ...this.time };\n }\n\n protected destroy() {\n this.#evtTool.destroy();\n }\n}\n\nexport function linearTimeFn(\n time: number,\n kf: TAnimationKeyFrame,\n opts: Required,\n): Partial {\n const offsetTime = time - opts.delay;\n if (offsetTime / opts.duration >= opts.iterCount) return {};\n\n const t = offsetTime % opts.duration;\n\n const process = offsetTime === opts.duration ? 1 : t / opts.duration;\n const idx = kf.findIndex((it) => it[0] >= process);\n if (idx === -1) return {};\n\n const startState = kf[idx - 1];\n const nextState = kf[idx];\n const nextFrame = nextState[1];\n if (startState == null) return nextFrame;\n const startFrame = startState[1];\n\n const rs: Partial = {};\n // 介于两个Frame状态间的进度\n const stateProcess =\n (process - startState[0]) / (nextState[0] - startState[0]);\n for (const prop in nextFrame) {\n const p = prop as keyof TAnimateProps;\n if (startFrame[p] == null) continue;\n // @ts-expect-error\n // eslint-disable-next-line\n rs[p] = (nextFrame[p] - startFrame[p]) * stateProcess + startFrame[p];\n }\n\n return rs;\n}\n","import { BaseSprite } from './base-sprite';\nimport { IClip } from '../clips';\nimport { Log } from '@webav/internal-utils';\nimport { changePCMPlaybackRate } from '../av-utils';\n\n/**\n * 包装 {@link IClip} 给素材扩展坐标、层级、透明度等信息，用于 {@link Combinator} 在后台合成视频\n *\n * 跟 {@link VisibleSprite} 非常相似，应用场景不同\n *\n * @example\n * const spr = new OffscreenSprite(\n * new MP4Clip((await fetch('')).body),\n * );\n * spr.opacity = 0.5 // 半透明\n * spr.rect.x = 100 // x 坐标偏移 100 像素\n * spr.time.offset = 10e6 // 视频第 10s 开始绘制该视频素材\n *\n * @see [视频合成](https://webav-tech.github.io/WebAV/demo/2_1-concat-video)\n */\nexport class OffscreenSprite extends BaseSprite {\n #clip: IClip;\n\n // 保持最近一帧，若 clip 在当前帧无数据，则绘制最近一帧\n #lastVf: VideoFrame | ImageBitmap | null = null;\n\n #destroyed = false;\n\n constructor(clip: IClip) {\n super();\n this.#clip = clip;\n this.ready = clip.ready.then(({ width, height, duration }) => {\n this.rect.w = this.rect.w === 0 ? width : this.rect.w;\n this.rect.h = this.rect.h === 0 ? height : this.rect.h;\n this.time.duration =\n this.time.duration === 0 ? duration : this.time.duration;\n });\n }\n\n /**\n * 绘制素材指定时刻的图像到 canvas 上下文，并返回对应的音频数据\n * @param time 指定时刻，微秒\n */\n async offscreenRender(\n ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,\n time: number,\n ): Promise<{\n audio: Float32Array[];\n done: boolean;\n }> {\n const ts = time * this.time.playbackRate;\n this.animate(ts);\n super._render(ctx);\n const { w, h } = this.rect;\n const { video, audio, state } = await this.#clip.tick(ts);\n let outAudio = audio ?? [];\n if (audio != null && this.time.playbackRate !== 1) {\n outAudio = audio.map((pcm) =>\n changePCMPlaybackRate(pcm, this.time.playbackRate),\n );\n }\n\n if (state === 'done') {\n return {\n audio: outAudio,\n done: true,\n };\n }\n\n const imgSource = video ?? this.#lastVf;\n if (imgSource != null) {\n ctx.drawImage(imgSource, -w / 2, -h / 2, w, h);\n }\n\n if (video != null) {\n this.#lastVf?.close();\n this.#lastVf = video;\n }\n\n return {\n audio: outAudio,\n done: false,\n };\n }\n\n async clone() {\n const spr = new OffscreenSprite(await this.#clip.clone());\n await spr.ready;\n this.copyStateTo(spr);\n return spr;\n }\n\n destroy(): void {\n if (this.#destroyed) return;\n this.#destroyed = true;\n\n Log.info('OffscreenSprite destroy');\n super.destroy();\n this.#lastVf?.close();\n this.#lastVf = null;\n this.#clip.destroy();\n }\n}\n","import { Log } from '@webav/internal-utils';\nimport { changePCMPlaybackRate } from '../av-utils';\nimport { IClip } from '../clips';\nimport { BaseSprite } from './base-sprite';\n\n/**\n * 包装 {@link IClip} 给素材扩展坐标、层级、透明度等信息，用于 {@link [AVCanvas](../../av-canvas/classes/AVCanvas.html)} 响应用户交互\n *\n * 跟 {@link OffscreenSprite} 非常相似，应用场景不同\n *\n * @example\n * const spr = new VisibleSprite(\n * new MP4Clip((await fetch('')).body),\n * );\n * spr.opacity = 0.5 // 半透明\n * spr.rect.x = 100 // x 坐标偏移 100 像素\n * spr.time.offset = 10e6 // 视频第 10s 开始绘制素材\n *\n * @see [视频剪辑](https://webav-tech.github.io/WebAV/demo/6_4-video-editor)\n *\n */\nexport class VisibleSprite extends BaseSprite {\n #clip: IClip;\n getClip() {\n return this.#clip;\n }\n\n /**\n * 元素是否可见，用于不想删除，期望临时隐藏 Sprite 的场景\n */\n visible = true;\n\n constructor(clip: IClip) {\n super();\n this.#clip = clip;\n this.ready = clip.ready.then(({ width, height, duration }) => {\n this.rect.w = this.rect.w === 0 ? width : this.rect.w;\n this.rect.h = this.rect.h === 0 ? height : this.rect.h;\n this.time.duration =\n this.time.duration === 0 ? duration : this.time.duration;\n });\n }\n\n // 保持最近一帧，若 clip 在当前帧无数据，则绘制最近一帧\n #lastVf: VideoFrame | ImageBitmap | null = null;\n #lastAudio: Float32Array[] = [];\n #ticking = false;\n #update(time: number) {\n if (this.#ticking) return;\n this.#ticking = true;\n\n this.#clip\n .tick(time * this.time.playbackRate)\n .then(({ video, audio }) => {\n if (video != null) {\n this.#lastVf?.close();\n this.#lastVf = video ?? null;\n }\n this.#lastAudio = audio ?? [];\n if (audio != null && this.time.playbackRate !== 1) {\n this.#lastAudio = audio.map((pcm) =>\n changePCMPlaybackRate(pcm, this.time.playbackRate),\n );\n }\n })\n .finally(() => {\n this.#ticking = false;\n });\n }\n\n /**\n * 提前准备指定 time 的帧\n */\n preFrame(time: number) {\n if (this.#lastTime === time) return;\n this.#lastVf?.close();\n this.#lastVf = null;\n this.#lastAudio = [];\n this.#update(time);\n this.#lastTime = time;\n }\n\n #lastTime = -1;\n /**\n * 绘制素材指定时刻的图像到 canvas 上下文，并返回对应的音频数据\n * @param time 指定时刻，微秒\n */\n render(\n ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,\n time: number,\n ): { audio: Float32Array[] } {\n this.animate(time);\n super._render(ctx);\n const { w, h } = this.rect;\n if (this.#lastTime !== time) this.#update(time);\n this.#lastTime = time;\n\n const audio = this.#lastAudio;\n this.#lastAudio = [];\n const video = this.#lastVf;\n if (video != null) ctx.drawImage(video, -w / 2, -h / 2, w, h);\n\n return { audio };\n }\n\n copyStateTo(target: T): void {\n super.copyStateTo(target);\n if (target instanceof VisibleSprite) {\n target.visible = this.visible;\n }\n }\n\n #destroyed = false;\n destroy(): void {\n if (this.#destroyed) return;\n this.#destroyed = true;\n\n Log.info('VisibleSprite destroy');\n super.destroy();\n this.#lastVf?.close();\n this.#lastVf = null;\n this.#clip.destroy();\n }\n}\n","import { Log, EventTool, file2stream, recodemux } from '@webav/internal-utils';\nimport { OffscreenSprite } from './sprite/offscreen-sprite';\nimport { sleep } from './av-utils';\nimport { DEFAULT_AUDIO_CONF } from './clips';\n\nexport interface ICombinatorOpts {\n width?: number;\n height?: number;\n bitrate?: number;\n fps?: number;\n bgColor?: string;\n videoCodec?: string;\n /**\n * false 合成的视频文件中排除音轨\n */\n audio?: false;\n /**\n * 向输出的视频中写入 meta tags 数据\n */\n metaDataTags?: Record;\n /**\n * 不安全，随时可能废弃\n */\n __unsafe_hardwareAcceleration__?: HardwarePreference;\n}\n\nlet COM_ID = 0;\n\n/**\n * 避免 VideoEncoder 队列中的 VideoFrame 过多，打爆显存\n */\nasync function letEncoderCalmDown(getQSize: () => number) {\n if (getQSize() > 50) {\n await sleep(15);\n await letEncoderCalmDown(getQSize);\n }\n}\n\n/**\n * 视频合成器；能添加多个 {@link OffscreenSprite}，根据它们位置、层级、时间偏移等信息，合成输出为视频文件\n * @see [视频合成](https://webav-tech.github.io/WebAV/demo/2_1-concat-video)\n * @see [视频配音](https://webav-tech.github.io/WebAV/demo/2_2-video-add-audio)\n * @example\n * const spr1 = new OffscreenSprite(\n * new MP4Clip((await fetch('')).body),\n * );\n * const spr2 = new OffscreenSprite(\n * new AudioClip((await fetch('