人体跟踪与骨骼关键点识别（C/C++）
===================

#### 约束与限制

从6.1.0(23)开始，人体跟踪与骨骼关键点识别能力支持部分Phone、部分Tablet设备、TV设备。请参考[硬件要求](https://developer.huawei.com/consumer/cn/doc/harmonyos-guides/arengine-preparations#硬件要求)判断设备是否支持人体骨骼点识别与跟踪特性（[ARENGINE\_FEATURE\_TYPE\_BODY](https://developer.huawei.com/consumer/cn/doc/harmonyos-references/arengine-capi-arengine#arengine_featuretype)）。

#### 接口说明

人体跟踪与骨骼关键点识别主要依赖ARBody，以下接口为AR Engine人体跟踪与骨骼关键点识别相关接口，详细接口和说明，请参考[AR Engine API参考](https://developer.huawei.com/consumer/cn/doc/harmonyos-references/arengine-capi-arengine)。

| 接口名 | 描述 |
| :-- | :-- |
| [HMS\_AREngine\_ARBody\_GetSkeletonConfidence](https://developer.huawei.com/consumer/cn/doc/harmonyos-references/arengine-capi-arengine#hms_arengine_arbody_getskeletonconfidence) | 获取人体对象每个骨骼点检测的置信度。 |
| [HMS\_AREngine\_ARBody\_GetSkeletonConnection](https://developer.huawei.com/consumer/cn/doc/harmonyos-references/arengine-capi-arengine#hms_arengine_arbody_getskeletonconnection) | 获取人体对象骨骼点之间的链接关系数据。 |
| [HMS\_AREngine\_ARBody\_GetSkeletonConnectionSize](https://developer.huawei.com/consumer/cn/doc/harmonyos-references/arengine-capi-arengine#hms_arengine_arbody_getskeletonconnectionsize) | 获取人体对象骨骼点之间的链接关系总数。 |
| [HMS\_AREngine\_ARBody\_GetSkeletonTypes](https://developer.huawei.com/consumer/cn/doc/harmonyos-references/arengine-capi-arengine#hms_arengine_arbody_getskeletontypes) | 获取识别出的人体对象骨骼点类型。 |
| [HMS\_AREngine\_ARBody\_GetSkeletonPointCount](https://developer.huawei.com/consumer/cn/doc/harmonyos-references/arengine-capi-arengine#hms_arengine_arbody_getskeletonpointcount) | 获取人体对象的骨骼点个数。 |
| [HMS\_AREngine\_ARBody\_GetSkeletonPointData2D](https://developer.huawei.com/consumer/cn/doc/harmonyos-references/arengine-capi-arengine#hms_arengine_arbody_getskeletonpointdata2d) | 当运行2D骨骼跟踪算法时，返回人体骨骼点的坐标数据。 |
| [HMS\_AREngine\_ARBody\_GetSkeletonPointIsValid](https://developer.huawei.com/consumer/cn/doc/harmonyos-references/arengine-capi-arengine#hms_arengine_arbody_getskeletonpointisvalid) | 获取人体对象骨骼点的坐标是否有效，返回有效性数组。 |
| [HMS\_AREngine\_ARBody\_GetBodyTrackId](https://developer.huawei.com/consumer/cn/doc/harmonyos-references/arengine-capi-arengine#hms_arengine_arbody_getbodytrackid) | 获取指定人体对象的标识。 |
| [HMS\_AREngine\_ARBody\_GetBodyTimeStamp](https://developer.huawei.com/consumer/cn/doc/harmonyos-references/arengine-capi-arengine#hms_arengine_arbody_getbodytimestamp) | 获取人体对象的检测时间点，表示触发检测人体对象距离启动相机的时间间隔，单位为ns。 |
| [HMS\_AREngine\_ARConfig\_SetBodyDetectedNum](https://developer.huawei.com/consumer/cn/doc/harmonyos-references/arengine-capi-arengine#hms_arengine_arconfig_setbodydetectednum) | 设置追踪人数。 |

#### 开发步骤

#### \[h2\]创建UI界面

创建一个UI界面，使用[XComponent](https://developer.huawei.com/consumer/cn/doc/harmonyos-references/ts-basic-components-xcomponent)组件显示相机预览画面，定时触发每一帧绘制。

// 此代码可参考示例代码：ARSample/entry/src/main/ets/pages/ARBody.ets。
import { Landmark } from '../body/ArBodyInterface';
import { arEngine, arViewController } from '@kit.AREngine';
import { resourceManager } from '@kit.LocalizationKit';
import { ARConfig } from '../utils/Utils';
import { display } from '@kit.ArkUI';
import { systemDateTime } from '@kit.BasicServicesKit';
import arEngineDemo from 'libarenginedemo.so';

@Builder
export function ARBodyBuilder() {
  ARBody();
}

interface ARBodyBoneLine {
  start: arEngine.ARBodyLandmarkType,
  end: arEngine.ARBodyLandmarkType,
}

@Entry
@ComponentV2
struct ARBody {
  private interval: number = -1;
  @Local context: Context = this.getUIContext().getHostContext() as Context;
  private idStr: string = systemDateTime.getTime(false).toString() + 'ARBody';
  private resMgr: resourceManager.ResourceManager = this.context.resourceManager;
  private params: ARConfig = new ARConfig();
  private isUpdate: boolean = false;
  private DEFAULT\_CONVERT\_FACTOR: number = 4 / 3;
  private boneLineNum: number = 26;
  private landMarkNum: number = 20;
  @Local displayWidth: number = display.getDefaultDisplaySync().width;
  @Local displayHeight: number = display.getDefaultDisplaySync().height;
  @Local rotation: number = display.getDefaultDisplaySync().rotation;
  @Local arContext?: arViewController.ARViewContext = undefined;
  @Local landmarks: Landmark\[\]\[\] = \[\];
  @Local boneLines: ARBodyBoneLine\[\]\[\] = \[\];
  @Local showPage: boolean = true;
  private uiContext: UIContext = this.getUIContext();

  @Builder
  drawBodyPerception() {
    Shape() {
      ForEach(this.landmarks, (landmarks: Landmark\[\], index: number) => {
        this.drawBodyLandmarks(index);
        this.drawBodyBones(index);
      })
    }
    .width(this.uiContext.px2vp(this.displayWidth))
    .height(this.uiContext.px2vp(this.displayHeight))
  }

  @Builder
  drawBodyLandmarks(bodyIndex: number) {
    ForEach(this.landmarks\[bodyIndex\], (landmark: Landmark, index: number) => {
      Circle({ width: 4, height: 4 })
        .position({
          x: this.uiContext.px2vp(landmark.x),
          y: this.uiContext.px2vp(landmark.y)
        })
        .fill(Color.White)
    })
  }

  @Builder
  drawBodyBones(bodyIndex: number) {
    if (this.boneLines.length > bodyIndex && this.landmarks\[bodyIndex\].length > 0) {
      ForEach(this.boneLines\[bodyIndex\], (bone: ARBodyBoneLine, index: number) => {
        this.drawBodyBoneLine(bone.start, bone.end, this.getLineColorByLineEnd(bone.end), bodyIndex);
      })
    }
  }

  @Builder
  drawBodyBoneLine(start: arEngine.ARBodyLandmarkType, end: arEngine.ARBodyLandmarkType, color: Color, index: number) {
    Line()
      .startPoint(\[this.uiContext.px2vp(this.landmarks\[index\]\[start - 1\]?.x),
        this.uiContext.px2vp(this.landmarks\[index\]\[start - 1\]?.y)\])
      .endPoint(\[this.uiContext.px2vp(this.landmarks\[index\]\[end - 1\]?.x),
        this.uiContext.px2vp(this.landmarks\[index\]\[end - 1\]?.y)\])
      .stroke(color)
      .strokeWidth(3)
  }

  private processLandMarks(res: Landmark\[\]): Landmark\[\]\[\] {
    if (display.getDefaultDisplaySync().width \* 3 < display.getDefaultDisplaySync().height \* 4) {
      this.displayWidth = display.getDefaultDisplaySync().width;
      this.displayHeight = this.displayWidth \* this.DEFAULT\_CONVERT\_FACTOR;
    } else {
      this.displayHeight = display.getDefaultDisplaySync().height;
      this.displayWidth = this.displayHeight \* this.DEFAULT\_CONVERT\_FACTOR;
    }

    let ret: Landmark\[\]\[\] = \[\];
    let num = 0;
    for (let i = 0; i < Math.min(this.params.maxDetectedBodyNum, res.length / this.landMarkNum); i++) {
      ret.push(\[\]);
      for (let j = 0; j < this.landMarkNum; j++) {
        ret\[i\].push({
          x: res\[num\].x \* this.displayWidth,
          y: res\[num\].y \* this.displayHeight
        })
        num++;
      }
    }
    return ret;
  }

  private getBoneLines(res: number\[\]): ARBodyBoneLine\[\]\[\] {
    let ret: ARBodyBoneLine\[\]\[\] = \[\];
    let num = 0;
    for (let i = 0; i < Math.min(this.params.maxDetectedBodyNum, res.length / this.boneLineNum); ++i) {
      ret.push(\[\]);
      for (let j = 0; j < this.boneLineNum; j += 2) {
        ret\[i\].push({
          start: res\[num\],
          end: res\[num + 1\]
        });
        num += 2;
      }
    }
    return ret;
  }

  private runArBodyCheck(): void {
    let config =
      new Int32Array(\[0, this.params.powerMode, 1, this.params.previewMode, 2, this.rotation, 9,
        this.params.cameraLensFacing, 10, this.params.maxDetectedBodyNum\]);
    arEngineDemo.init(this.resMgr);
    arEngineDemo.start(this.idStr, config);
    this.interval = setInterval(() => {
      if (!this.isUpdate) {
        return;
      }
      this.rotation = display.getDefaultDisplaySync().rotation;
      arEngineDemo.update(this.idStr, this.rotation);
      this.landmarks = this.processLandMarks(arEngineDemo.getLandmark(this.idStr));
      const boneLines: number\[\] = arEngineDemo.getBoneLine(this.idStr).skeletonConnections;
      this.boneLines = this.getBoneLines(boneLines)
    }, 66);
  }

  private getLineColorByLineEnd(lineEnd: arEngine.ARBodyLandmarkType): Color {
    switch (lineEnd) {
      case arEngine.ARBodyLandmarkType.LEFT\_ELBOW:
      case arEngine.ARBodyLandmarkType.LEFT\_WRIST:
      case arEngine.ARBodyLandmarkType.LEFT\_KNEE:
      case arEngine.ARBodyLandmarkType.LEFT\_ANKLE:
        return Color.Green;
      case arEngine.ARBodyLandmarkType.RIGHT\_ELBOW:
      case arEngine.ARBodyLandmarkType.RIGHT\_WRIST:
      case arEngine.ARBodyLandmarkType.RIGHT\_KNEE:
      case arEngine.ARBodyLandmarkType.RIGHT\_ANKLE:
        return Color.Blue;
      case arEngine.ARBodyLandmarkType.RIGHT\_SHOULDER:
      case arEngine.ARBodyLandmarkType.RIGHT\_HIP:
      case arEngine.ARBodyLandmarkType.LEFT\_SHOULDER:
      case arEngine.ARBodyLandmarkType.LEFT\_HIP:
      default:
        return Color.Orange;
    }
  }

  build() {
    NavDestination() {
      RelativeContainer() {
        Stack() {
          XComponent({ id: this.idStr, type: XComponentType.SURFACE, libraryname: 'arenginedemo' })
            .width(this.uiContext.px2vp(this.displayWidth))
            .height(this.uiContext.px2vp(this.displayHeight))
            .visibility(this.showPage ? Visibility.Visible : Visibility.None)
            .alignRules({
              center: { anchor: '\_\_container\_\_', align: VerticalAlign.Center },
              middle: { anchor: '\_\_container\_\_', align: HorizontalAlign.Center }
            })
          this.drawBodyPerception();
        }
        .width('100%')
        .height('100%')
      }
    }
    .onAppear(async () => {
      this.runArBodyCheck();
    })
    .onWillDisappear(() => {
      clearInterval(this.interval);
      arEngineDemo.stop(this.idStr);
    })
    .onShown(() => {
      this.isUpdate = true;
      arEngineDemo.show(this.idStr);
    })
    .onHidden(() => {
      this.isUpdate = false;
      arEngineDemo.hide(this.idStr);
    })
    .onReady(ctx => {
      this.params = ctx.pathInfo.param as ARConfig;
    })
    .hideTitleBar(true)
    .hideBackButton(true)
    .hideToolBar(true)
    .width('100%')
    .height('100%')
  }
}

#### \[h2\]引入AR Engine

开发者可参考管理AR会话章节的[引入AR Engine](https://developer.huawei.com/consumer/cn/doc/harmonyos-guides/arengine-c-arsession#引入ar-engine)。

#### \[h2\]创建AR会话

创建AR会话，配置人体跟踪模式。使用人体跟踪与骨骼关键点识别能力时请使用[HMS\_AREngine\_ARSession\_Create\_Human\_Perception](https://developer.huawei.com/consumer/cn/doc/harmonyos-references/arengine-capi-arengine#hms_arengine_arsession_create_human_perception)创建AR会话。

AREngine\_ARSession \*arSession = nullptr;
// 创建AR会话。
HMS\_AREngine\_ARSession\_Create\_Human\_Perception(nullptr, nullptr, &arSession);
AREngine\_ARConfig \*arConfig = nullptr;
// 创建AR会话配置器。
HMS\_AREngine\_ARConfig\_Create(arSession, &arConfig);
// 设置人体跟踪模式的AR类型。
HMS\_AREngine\_ARConfig\_SetARType(arSession, arConfig, ARENGINE\_TYPE\_BODY);
// 设置追踪人数，当前支持1或2
HMS\_AREngine\_ARConfig\_SetBodyDetectedNum(arSession, arConfig, 1);
// 配置器设置给AR会话。
HMS\_AREngine\_ARSession\_Configure(arSession, arConfig);

#### \[h2\]创建可跟踪对象列表

创建一个可跟踪对象列表targetList，用于存放AR Engine运行过程中检测到的所有可跟踪对象。

AREngine\_ARTrackableList \*targetList = nullptr;
HMS\_AREngine\_ARTrackableList\_Create(arSession, &targetList);

#### \[h2\]获取人体追踪对象

调用[HMS\_AREngine\_ARSession\_GetAllTrackables](https://developer.huawei.com/consumer/cn/doc/harmonyos-references/arengine-capi-arengine#hms_arengine_arsession_getalltrackables)函数，检测当前环境中的所有人体追踪对象，并将结果存放在targetList中。

HMS\_AREngine\_ARSession\_GetAllTrackables(arSession, ARENGINE\_TRACKABLE\_BODY, targetList);

#### \[h2\]获取可跟踪对象数量

调用[HMS\_AREngine\_ARTrackableList\_GetSize](https://developer.huawei.com/consumer/cn/doc/harmonyos-references/arengine-capi-arengine#hms_arengine_artrackablelist_getsize)函数获取当前可跟踪对象数量，结果存放在targetSize中。

int32\_t targetSize = 0;
HMS\_AREngine\_ARTrackableList\_GetSize(arSession, targetList, &targetSize);

#### \[h2\]获取骨骼点相关信息

for (int i = 0; i < targetSize; i++) {
    AREngine\_ARTrackable \*arTrackable = nullptr;
    // 从可跟踪列表中获取指定index的对象。
    HMS\_AREngine\_ARTrackableList\_AcquireItem(arSession, targetList, i, &arTrackable);
    // 转换为人体对象AREngine\_ARBody。
    AREngine\_ARBody \*arBody = reinterpret\_cast<AREngine\_ARBody \*>(arTrackable);
    int32\_t pointCnt = 0;
    // 获取人体对象的骨骼点个数。
    HMS\_AREngine\_ARBody\_GetSkeletonPointCount(arSession, arBody, &pointCnt);
    const float \*skeletonPoints2D = nullptr;
    // 获取人体骨骼点的2D坐标数据。
    HMS\_AREngine\_ARBody\_GetSkeletonPointData2D(arSession, arBody, &skeletonPoints2D);
    std::vector<float> skeletonPoints2Ds;
    for (int j = 0; j < pointCnt \* 2; j++) {
        skeletonPoints2Ds.push\_back(skeletonPoints2D\[j\]);
    }
    coord.push\_back(skeletonPoints2Ds);
    const float \*skeletonConfidences = nullptr;
    // 获取人体对象每个骨骼点检测的置信度。
    HMS\_AREngine\_ARBody\_GetSkeletonConfidence(arSession, arBody, &skeletonConfidences);
    std::vector<float> skeletonConfidence2Ds;
    for (int j = 0; j < pointCnt; j++) {
        skeletonConfidence2Ds.push\_back(skeletonConfidences\[j\]);
    }
    confidence.push\_back(skeletonConfidence2Ds);
    const int32\_t \*isValids = nullptr;
    // 获取人体对象骨骼点的坐标是否有效。
    HMS\_AREngine\_ARBody\_GetSkeletonPointIsValid(arSession, arBody, &isValids);
    std::vector<int32\_t> skeletonValid2Ds;
    for (int j = 0; j < pointCnt; j++) {
        skeletonValid2Ds.push\_back(isValids\[j\]);
    }
    valid.push\_back(skeletonValid2Ds);
    int32\_t outBodyTrackId = 0;
    // 获取指定人体对象的标识。
    HMS\_AREngine\_ARBody\_GetBodyTrackId(arSession, arBody, &outBodyTrackId);
    int64\_t timeStampNanoSec = 0;
    // 获取人体对象的检测时间点，表示触发检测人体对象距离启动相机的时间间隔，单位为ns。
    HMS\_AREngine\_ARBody\_GetBodyTimeStamp(arSession, arBody, &timeStampNanoSec);
    int32\_t connectionSize = 0;
    // 获取人体对象骨骼点之间的链接关系总数。
    HMS\_AREngine\_ARBody\_GetSkeletonConnectionSize(arSession, arBody, &connectionSize);
    const AREngine\_ARBodySkeletonType \*skeletonConnections2D = nullptr;
    // 获取人体对象骨骼点之间的链接关系数据。
    HMS\_AREngine\_ARBody\_GetSkeletonConnection(arSession, arBody, &skeletonConnections2D);
    const AREngine\_ARBodySkeletonType \*skeletonTypes = nullptr;
    // 获取识别出的人体对象骨骼点类型。
    HMS\_AREngine\_ARBody\_GetSkeletonTypes(arSession, arBody, &skeletonTypes);
    std::vector<int32\_t> skeletonValid2Ds;
    for (int j = 0; j < connectionSize \* 2; j++) {
        skeletonValid2Ds.push\_back(skeletonConnections2D\[j\]);
    }
    std::vector<int32\_t> skeletonTypesVec;
    for (int j = 0; j < sizeof(std::underlying\_type\_t<AREngine\_ARBodySkeletonType>); j++) {
        skeletonTypesVec.push\_back(skeletonConnections2D\[j\]);
    }
    connections.push\_back(skeletonValid2Ds);
    types.push\_back(skeletonTypesVec);
}

#### \[h2\]销毁可跟踪对象列表

可跟踪对象列表targetList不再使用后需销毁：

HMS\_AREngine\_ARTrackableList\_Destroy(targetList);