自定义相机如何隔一段时间获取并保存预览帧的图像
=======================

#### 问题现象

如何使用自定义相机打开前置摄像头，隔一段时间截取视频帧，编码成图片文件后保存到沙箱，以供后续发送图像数据进行识别？

#### 背景知识

*   [ImageReceiver](https://developer.huawei.com/consumer/cn/doc/harmonyos-references/arkts-apis-image-imagereceiver)为图像接收类，用于获取组件surface id，接收最新的图片和读取下一张图片，以及释放ImageReceiver实例。
*   [packToData](https://developer.huawei.com/consumer/cn/doc/harmonyos-references/arkts-apis-image-imagepacker#packtodata13)可用于图片压缩或重新编码，可以使用此接口将PixelMap类型的数据编码成图片文件保存至应用沙箱。

#### 解决方案

1.  首先使用[ImageReceiver](https://developer.huawei.com/consumer/cn/doc/harmonyos-references/arkts-apis-image-imagereceiver)接收预览帧图片，具体方法可参考[双路预览](https://developer.huawei.com/consumer/cn/doc/harmonyos-guides/camera-dual-channel-preview)。
2.  将获取到的预览帧图片编码成图片保存至应用沙箱，参考代码如下：
    
    _/\*\*_
     _\* 将PixelMap对象的图片数据保存至应用沙箱路径_
     _\* @param pixelMap 需要保存的pixelMap对象_
     _\* @returns_
     _\*/_
    async packToData(pixelMap: PixelMap) {
      try {
        let context = this.getUIContext().getHostContext() as common.UIAbilityContext;
        let cacheDir: string = context.cacheDir;
        let now = Date.now();
        let fileUrl = cacheDir + \`/${now}\_packToData.jpg\`;
        if (fs.accessSync(fileUrl)) {
          fs.unlinkSync(fileUrl);
        }
        let packOpts: image.PackingOption = { format: 'image/jpeg', quality: 98 };
        const imagePackerApi: image.ImagePacker = image.createImagePacker();
        imagePackerApi.packToData(pixelMap, packOpts)
          .then((data: ArrayBuffer) => {
            let file = fs.openSync(fileUrl, fs.OpenMode.READ\_WRITE | fs.OpenMode.CREATE);
            fs.write(file.fd, data).then((writeLen: number) => {
              hilog.info(DOMAIN, TAG, \`packToData succeed and size is: ${writeLen}, path is ${fileUrl}}\`);
            }).catch((err: BusinessError) => {
              hilog.error(DOMAIN, TAG, \`packToData failed with error message: ${err.message}, error code: ${err.code}\`);
            }).finally(() => {
              fs.closeSync(file);
            });
          }).catch((error: BusinessError) => {
          hilog.error(DOMAIN, TAG, \`packToData error，code ${error.code}, message is ${error.message}\`);
        });
      } catch (err) {
        let e: BusinessError = err;
        hilog.error(DOMAIN, TAG,
          \`write file by pixelmap error is: ${JSON.stringify(e.message)} and stack: ${JSON.stringify(e.stack)}\`);
      }
    }
    
3.  使用[setInterval](https://developer.huawei.com/consumer/cn/doc/harmonyos-references/js-apis-timer#setinterval)定时器，设置时间定时执行步骤2：
    
    onPageShow(): void {
      hilog.info(DOMAIN, TAG, 'onPageShow');
      this.initImageReceiver();
      if (this.xComponentSurfaceId !== '') {
        this.initCamera();
      }
     _ // 将提取到的视频帧编码进应用沙箱_
      setInterval(() => {
        this.packToData(this.previewImage!);
      }, this.period \* 1000);
    }
    

完整代码如下：

import { camera } from '@kit.CameraKit';
import { image } from '@kit.ImageKit';
import { BusinessError } from '@kit.BasicServicesKit';
import { fileIo as fs } from '@kit.CoreFileKit';
import { abilityAccessCtrl, common, Permissions } from '@kit.AbilityKit';
import { hilog } from '@kit.PerformanceAnalysisKit';


interface CameraResources {
  videoOutput?: camera.VideoOutput;
  cameraInput?: camera.CameraInput;
  previewOutput1?: camera.PreviewOutput;
  previewOutput2?: camera.PreviewOutput;
  session?: camera.VideoSession;
}


const DOMAIN: number = 0x0000;
const TAG: string = '\[CameraDemo\]';


@Entry
@Component
struct Index {
  @State isShow: boolean = false;
  private previewImage: image.PixelMap | undefined = undefined;
  private imageReceiver: image.ImageReceiver | undefined = undefined;
  private imageReceiverSurfaceId: string = '';
  private xComponentCtl: XComponentController = new XComponentController();
  private xComponentSurfaceId: string = '';
  private imageWidth: number = 1920;
  private imageHeight: number = 1080;
  private cameraManager: camera.CameraManager | undefined = undefined;
  private cameras: Array<camera.CameraDevice> | undefined = \[\];
  private uiContext: UIContext = this.getUIContext();
  private context: Context | undefined = this.uiContext.getHostContext();
  private cameraPermission: Permissions = 'ohos.permission.CAMERA'; _// 申请权限_
  private period: number = 3; _// 定义提取视频帧时间间隔_
  private cameraResources: CameraResources = {};


 _ /\*\*_
   _\* 向用户申请权限_
   _\* @returns_
   _\*/_
  async requestPermissionsFn(): Promise<void> {
    let atManager = abilityAccessCtrl.createAtManager();
    if (this.context) {
      let res = await atManager.requestPermissionsFromUser(this.context, \[this.cameraPermission\]);
      if (!res || !res.permissions || res.permissions.length === 0) {
        hilog.error(DOMAIN, TAG, 'requestPermissionsFromUser interface call fails');
        return;
      }
      if (res.permissions.length !== res.authResults.length) {
        hilog.error(DOMAIN, TAG, 'Authentication result mismatch');
        return;
      }
      res.permissions.forEach((value: string, index: number, permissions: string\[\]) => {
        if (this.cameraPermission.toString() === value && res.authResults\[index\] === 0) {
          this.isShow = true;
        }
        permissions.forEach((permission: string) => {
          hilog.info(DOMAIN, TAG, \`${permission} granted\`);
        });
      });
    }
  }


  aboutToAppear(): void {
    this.requestPermissionsFn();
  }


  onPageShow(): void {
    hilog.info(DOMAIN, TAG, 'onPageShow');
    this.initImageReceiver();
    if (this.xComponentSurfaceId !== '') {
      this.initCamera();
    }
   _ // 将提取到的视频帧编码进应用沙箱_
    setInterval(() => {
      this.packToData(this.previewImage!);
    }, this.period \* 1000);
  }


  onPageHide(): void {
    hilog.info(DOMAIN, TAG, 'onPageHide');
    this.releaseCamera();
  }


  build() {
    Column() {
      if (this.isShow) {
        XComponent({
          id: 'componentId',
          type: XComponentType.SURFACE,
          controller: this.xComponentCtl
        })
          .onLoad(async () => {
            hilog.info(DOMAIN, TAG, 'onLoad is called');
            this.xComponentSurfaceId = this.xComponentCtl.getXComponentSurfaceId(); _// 获取组件surfaceId_
          _  // 初始化相机，组件实时渲染每帧预览流数据_
            this.initCamera();
          })
          .width(this.uiContext.px2vp(this.imageHeight))
          .height(this.uiContext.px2vp(this.imageWidth))
      }
    }
    .justifyContent(FlexAlign.Center)
    .height('100%')
    .width('100%')
  }


 _ /\*\*_
   _\* 获取ImageReceiver的SurfaceId_
   _\* @returns_
   _\*/_
  async initImageReceiver(): Promise<void> {
    if (!this.imageReceiver) {
    _  // 创建ImageReceiver_
      let size: image.Size = { width: this.imageWidth, height: this.imageHeight };
      try {
        this.imageReceiver = image.createImageReceiver(size, image.ImageFormat.JPEG, 8);
      } catch (error) {
        let err = error as BusinessError;
        hilog.error(DOMAIN, TAG, \`Init image receiver failed. error code: ${err.code}\`);
        return;
      }
    _  // 获取第一路流SurfaceId_
      this.imageReceiverSurfaceId = await this.imageReceiver.getReceivingSurfaceId();
      hilog.info(DOMAIN, TAG, \`initImageReceiver imageReceiverSurfaceId:${this.imageReceiverSurfaceId}\`);
    _  // 注册监听处理预览流每帧图像数据_
      this.onImageArrival(this.imageReceiver);
    }
  }


 _ // Image格式与PixelMap格式映射关系_
  private formatToPixelMapFormatMap = new Map<number, image.PixelMapFormat>(\[
    \[12, image.PixelMapFormat.RGBA\_8888\],
    \[25, image.PixelMapFormat.NV21\],
    \[35, image.PixelMapFormat.YCBCR\_P010\],
    \[36, image.PixelMapFormat.YCRCB\_P010\]
  \]);
  _// PixelMapFormat格式的单个像素点大小映射关系_
  private pixelMapFormatToSizeMap = new Map<image.PixelMapFormat, number>(\[
    \[image.PixelMapFormat.RGBA\_8888, 4\],
    \[image.PixelMapFormat.NV21, 1.5\],
    \[image.PixelMapFormat.YCBCR\_P010, 3\],
    \[image.PixelMapFormat.YCRCB\_P010, 3\]
  \]);


 _ /\*\*_
   _\* 将PixelMap对象的图片数据保存至应用沙箱路径_
   _\* @param pixelMap 需要保存的pixelMap对象_
   _\* @returns_
   _\*/_
  async packToData(pixelMap: PixelMap) {
    try {
      let context = this.getUIContext().getHostContext() as common.UIAbilityContext;
      let cacheDir: string = context.cacheDir;
      let now = Date.now();
      let fileUrl = cacheDir + \`/${now}\_packToData.jpg\`;
      if (fs.accessSync(fileUrl)) {
        fs.unlinkSync(fileUrl);
      }
      let packOpts: image.PackingOption = { format: 'image/jpeg', quality: 98 };
      const imagePackerApi: image.ImagePacker = image.createImagePacker();
      imagePackerApi.packToData(pixelMap, packOpts)
        .then((data: ArrayBuffer) => {
          let file = fs.openSync(fileUrl, fs.OpenMode.READ\_WRITE | fs.OpenMode.CREATE);
          fs.write(file.fd, data).then((writeLen: number) => {
            hilog.info(DOMAIN, TAG, \`packToData succeed and size is: ${writeLen}, path is ${fileUrl}}\`);
          }).catch((err: BusinessError) => {
            hilog.error(DOMAIN, TAG, \`packToData failed with error message: ${err.message}, error code: ${err.code}\`);
          }).finally(() => {
            fs.closeSync(file);
          });
        }).catch((error: BusinessError) => {
        hilog.error(DOMAIN, TAG, \`packToData error，code ${error.code}, message is ${error.message}\`);
      });
    } catch (err) {
      let e: BusinessError = err;
      hilog.error(DOMAIN, TAG,
        \`write file by pixelmap error is: ${JSON.stringify(e.message)} and stack: ${JSON.stringify(e.stack)}\`);
    }
  }


 _ /\*\*_
   _\* 注册ImageReceiver图像监听_
   _\* @param receiver_
   _\* @returns_
   _\*/_
  onImageArrival(receiver: image.ImageReceiver): void {
   _ // 注册imageArrival监听_
    receiver.on('imageArrival', () => {
      hilog.info(DOMAIN, TAG, 'image arrival');
    _  // 获取图像_
      receiver.readNextImage((err: BusinessError, nextImage: image.Image) => {
        if (err || nextImage === undefined) {
          hilog.error(DOMAIN, TAG, 'readNextImage failed');
          return;
        }
       _ // 解析图像内容_
        nextImage.getComponent(image.ComponentType.JPEG, async (err: BusinessError, imgComponent: image.Component) => {
          if (err || imgComponent === undefined) {
            hilog.error(DOMAIN, TAG, 'getComponent failed');
          }
          if (imgComponent.byteBuffer) {
            let width = nextImage.size.width; _// 获取图片的宽_
            let height = nextImage.size.height; _// 获取图片的高_
            let stride = imgComponent.rowStride; _// 获取图片的stride_
            let imageFormat = nextImage.format; _// 获取图片的format_
            let pixelMapFormat = this.formatToPixelMapFormatMap.get(imageFormat) ?? image.PixelMapFormat.NV21;
            let mSize = this.pixelMapFormatToSizeMap.get(pixelMapFormat) ?? 1.5;
            hilog.info(DOMAIN, TAG, \`getComponent with width:${width} height:${height} stride:${stride}\`);
          _  // pixelMap创建时使用的size、srcPixelFormat需要与相机预览输出流previewProfile中的size、format保持一致，此处format以NV21格式为例_
            _// stride与width一致_
            let pixelMap: image.PixelMap;
            if (stride === width) {
              pixelMap = await image.createPixelMap(imgComponent.byteBuffer, {
                size: { height: height, width: width },
                srcPixelFormat: pixelMapFormat,
              });
              this.previewImage = pixelMap;
            } else {
             _ // stride与width不一致_
              const dstBufferSize = width \* height \* mSize; _// 以NV21为例（YUV\_420\_SP格式的图片）YUV\_420\_SP内存计算公式：长x宽+(长x宽)/2_
              const dstArr = new Uint8Array(dstBufferSize);
              for (let j = 0; j < height \* mSize; j++) {
                const srcBuf = new Uint8Array(imgComponent.byteBuffer, j \* stride, width);
                dstArr.set(srcBuf, j \* width);
              }
              pixelMap = await image.createPixelMap(dstArr.buffer, {
                size: { height: height, width: width },
                srcPixelFormat: pixelMapFormat,
              });
              this.previewImage = pixelMap;
            }
          } else {
            hilog.error(DOMAIN, TAG, 'byteBuffer is null');
          }
          _// 确保当前buffer没有在使用的情况下，可进行资源释放_
          _// 如果对buffer进行异步操作，需要在异步操作结束后再释放该资源（nextImage.release()）_
          nextImage.release();
          hilog.info(DOMAIN, TAG, 'image process done');
        });
      });
    });
  }


 _ /\*\*_
   _\* 初始化相机流程_
   _\* @returns_
   _\*/_
  async initCamera(): Promise<void> {
    hilog.info(DOMAIN, TAG,
      \`initCamera imageReceiverSurfaceId:${this.imageReceiverSurfaceId} xComponentSurfaceId:${this.xComponentSurfaceId}\`);
    try {
     _ // 获取相机管理器实例_
      this.cameraManager = camera.getCameraManager(this.context);
      if (!this.cameraManager) {
        hilog.error(DOMAIN, TAG, 'getCameraManager call failed');
        return;
      }
   _   // 获取当前设备支持的相机device列表_
      this.cameras = this.cameraManager.getSupportedCameras();
      if (!this.cameras || this.cameras.length === 0) {
        hilog.error(DOMAIN, TAG, 'getSupportedCameras call failed');
        return;
      }
      let cameraDevice: camera.CameraDevice | undefined = undefined;
      this.cameras.forEach((device: camera.CameraDevice) => {
        if (device.cameraPosition === camera.CameraPosition.CAMERA\_POSITION\_FRONT) {
          cameraDevice = device;
          return;
        }
      });


    _  // 未找到前置摄像头_
      if (cameraDevice === undefined) {
        hilog.error(DOMAIN, TAG, 'Front camera device not found!');
        return;
      }


     _ // 选择一个相机device，创建cameraInput输出对象_
      this.cameraResources.cameraInput = this.cameraManager.createCameraInput(cameraDevice);
      if (!this.cameraResources.cameraInput) {
        hilog.error(DOMAIN, TAG, 'createCameraInput call failed');
        return;
      }
     _ // 打开相机_
      await this.cameraResources.cameraInput.open();
   _   // 获取相机device支持的profile_
      let capability: camera.CameraOutputCapability =
        this.cameraManager.getSupportedOutputCapability(cameraDevice, camera.SceneMode.NORMAL\_VIDEO);
      if (!capability) {
        hilog.error(DOMAIN, TAG, 'getSupportedOutputCapability call failed');
        this.releaseCamera();
        return;
      }
      let minRatioDiff: number = 0.1;
      let surfaceRatio: number = this.imageWidth / this.imageHeight; _// 最接近16:9宽高比_
      let previewProfile: camera.Profile = capability.previewProfiles\[0\];
     _ // 应用开发者根据实际业务需求选择一个支持的预览流previewProfile_
      _// 此处以选择CAMERA\_FORMAT\_YUV\_420\_SP（NV21）格式、满足限定条件分辨率的预览流previewProfile为例_
      for (let index = 0; index < capability.previewProfiles.length; index++) {
        const tempProfile = capability.previewProfiles\[index\];
        let tempRatio = tempProfile.size.width >= tempProfile.size.height ?
          tempProfile.size.width / tempProfile.size.height : tempProfile.size.height / tempProfile.size.width;
        let currentRatio = Math.abs(tempRatio - surfaceRatio);
        if (currentRatio <= minRatioDiff && tempProfile.format === camera.CameraFormat.CAMERA\_FORMAT\_YUV\_420\_SP) {
          previewProfile = tempProfile;
          break;
        }
      }
      this.imageWidth = previewProfile.size.width; _// 更新xComponent组件的宽_
      this.imageHeight = previewProfile.size.height; _// 更新xComponent组件的高_
      hilog.info(DOMAIN, TAG, \`initCamera imageWidth: ${this.imageWidth} imageHeight: ${this.imageHeight}\`);
     _ // 使用imageReceiverSurfaceId创建第一路预览_
      this.cameraResources.previewOutput1 =
        this.cameraManager.createPreviewOutput(previewProfile, this.imageReceiverSurfaceId);
      if (!this.cameraResources.previewOutput1) {
        hilog.error(DOMAIN, TAG, 'initCamera createPreviewOutput1');
        this.releaseCamera();
        return;
      }
    _  // 使用xComponentSurfaceId创建第二路预览_
      this.cameraResources.previewOutput2 =
        this.cameraManager.createPreviewOutput(previewProfile, this.xComponentSurfaceId);
      if (!this.cameraResources.previewOutput2) {
        hilog.error(DOMAIN, TAG, 'initCamera createPreviewOutput2');
        this.releaseCamera();
        return;
      }
     _ // 创建录像模式相机会话_
      let session = this.cameraManager.createSession(camera.SceneMode.NORMAL\_VIDEO);
      if (!session) {
        hilog.error(DOMAIN, TAG, 'session is null');
        this.releaseCamera();
        return;
      }
      this.cameraResources.session = session as camera.VideoSession;
     _ // 开始配置会话_
      this.cameraResources.session.beginConfig();
      _// 添加相机设备输入_
      this.cameraResources.session.addInput(this.cameraResources.cameraInput);
     _ // 添加第一路预览流输出_
      this.cameraResources.session.addOutput(this.cameraResources.previewOutput1);
    _  // 添加第二路预览流输出_
      this.cameraResources.session.addOutput(this.cameraResources.previewOutput2);
     _ // 提交会话配置_
      await this.cameraResources.session.commitConfig();
     _ // 开始启动已配置的输入输出流_
      await this.cameraResources.session.start();
    } catch (error) {
      hilog.error(DOMAIN, TAG, \`initCamera fail: ${JSON.stringify(error)}\`);
      this.releaseCamera();
    }
  }

  _/\*\*_
   _\* 释放相机相关资源_
   _\* @returns_
   _\*/_
  async releaseCamera(): Promise<void> {
    hilog.info(DOMAIN, TAG, 'releaseCamera E');
    try {
      _// 停止当前会话_
      await this.cameraResources.session?.stop();
    } catch (error) {
      hilog.error(DOMAIN, TAG, \`session.stop call failed, error: ${JSON.stringify(error)}\`);
    }
    try {
      _// 释放相机输入流_
      await this.cameraResources.cameraInput?.close();
    } catch (error) {
      hilog.error(DOMAIN, TAG, \`camera close fail: ${JSON.stringify(error)}\`);
    }
    try {
     _ // 释放预览输出流_
      await this.cameraResources.previewOutput1?.release();
    } catch (error) {
      hilog.error(DOMAIN, TAG, \`previewOutput1 release fail: ${JSON.stringify(error)}\`);
    }
    try {
    _  // 释放拍照输出流_
      await this.cameraResources.previewOutput2?.release();
    } catch (error) {
      hilog.error(DOMAIN, TAG, \`previewOutput2 release fail: ${JSON.stringify(error)}\`);
    }
    try {
     _ // 释放会话_
      await this.cameraResources.session?.release();
    } catch (error) {
      hilog.error(DOMAIN, TAG, \`session release fail: ${JSON.stringify(error)}\`);
    }
  }
}

注意使用相机功能前，需要在module.json5文件中申请相机权限：

{
  "name": "ohos.permission.CAMERA",
  "reason": "$string:reason\_camera",
  "usedScene": {
    "abilities": \[
      "EntryAbility"
    \]
  }
},