Android 实例：跳绳

开发基础

MediaPipe 基础

人体姿态说明

开发语言

Java, Kotlin

下载 JDK

开发工具

Android Studio 下载 Android Studio 下载 git

开发设备

PC + 手机（Android系统）

源码地址

https://github.com/UfutxAI/jumping

基于官方代码的解析

代码结构

  • app/src/main
    • java  //代码文件夹
      • fragement
        • CameraFragment  //摄像头页面
        • GalleryFragment  //图片页面
        • PermissionsFragment  //授权页面
      • MainActivity  //主页面（入口文件）
      • MainViewModel  //页面设置的参数文件
      • OverlayView  //画人格框架的文件
      • PoseLandmarkerHelper  //姿势处理文件
    • res  //资源文件夹
      • layout  //布局页面文件夹
        • activity_main.xml  //主页面，主布局
        • fragment_camera.xml  //分页面，摄像头布局
        • fragment_gallery.xml  //分页面，图片或者视频布局
        • info_bottom_sheet.xml  //分页面，参数配置
      • values
        • colors.xml  //颜色参数配置文件
        • dimens.xml  //布局参数配置文件
        • strings.xml  //文字参数配置文件
        • styles.xml  //样式主题参数配置文件

代码说明

项目启动及文件加载流程

CameraFragment运行流程

CameraFragment继承了两个类

  1. Fragment 作为多页面 Activity中的一个页面（Fragment）运行
  2. PoseLandmarkerHelper 姿态识别类，主要作用是
    1. 初始化【姿态识别器】，包括
      1. 设置处理器类型：CPU / GPU，手机默认是 CPU
      2. 设置识别模型文件（Task 文件）
      3. 设置识别阈值
      4. 设置识别模式（图片，视频，实时视频）
      5. 如果是实时视频，设置实时识别结果的回调函数用于处理实时图片数据
    2. 识别图片
    3. 识别视频，将视频按时间切片成图片
    4. 识别实时视频，将 ImageProxyl 生成图片
    5. 统一交到 系统 SDK PoseLandmarker的detectAsync(Image, TimestampMilliSecond)进行识别
  3. 启动 Fragement
  4. 初始化页面布局
  5. 启动 newSingleThreadExecutor 创建一个单线程实例化PoseLandmarkerHelper
  6. 初始化摄像头，并且将实时的ImageProxy传到 PoseLandmarkerHelper 进行处理

跳绳的算法

CameraFragment 文件说明

//实时视频的处理，数据量很大，所以需要间隔采取识别数据

private var allY = arrayListOf<ArrayList<Float>>() //存储所需要的结果点的 Y 轴值

private var jumpingCount = 0 // 结果个数    
private var jumpingHint = "" 0 //跳绳的提示

private val resultInterval= 2 //间隔 3 个才取一个数据
private var resultCount = 0 //与间隔



private var oneMinuteMode = false //当前是否是在 1 分钟计数
private var oneMinuteLeft = "" //1 分钟计数，还剩下多少秒

private val visibilityThreshold = 0.8f //节点可见度 1 = 100%
private val continuityThreshold = 0.8f // 数据连续性比率 1 = 100%

···
//获取到检测结果之后
override fun onResults(resultBundle: PoseLandmarkerHelper.ResultBundle){
    resultCount += 1
    if(resultCount % resultInterval == 0){
        var nodeNose = resultBundle.results.first().landmarks().first()[0]  //鼻子
        var nodeWaistLeft= resultBundle.results.first().landmarks().first()[23]  //左腰点
        var nodeWaistRight = resultBundle.results.first().landmarks().first()[24]  //右腰点
        var nodeAnkleLeft = resultBundle.results.first().landmarks().first()[27]  //左脚踝
        var nodeAnkleRight = resultBundle.results.first().landmarks().first()[28]  //右脚踝

        //添加鼻子，双腰，双脚踝 5个点作为跳上跳下的处理
        var nodeList = arrayListOf(nodeNose, nodeWaistLeft, nodeAnkleRight, nodeAnkleLeft, nodeWaistRight)
        var nodeY = ArrayList<Float>()
        for (idx in 0 until nodeList.size){
            if ((nodeList[idx].visibility() != null ) && nodeList[idx].visibility().get() > visibilityThreshold){
                startCounting = startCounting && true
                nodeY.add(nodeList[idx].y())
            }else{
                countHint = "需要全身都在视频当中才行"
                startCounting = startCounting && false
                break;
            }
        }
        jumpingHint = countHint
        if (startCounting){
            resultCount += 1
            if(resultCount % resultInterval == 0){
                allY.add(nodeY)
                Log.e(TAG, "Add new Value: $nodeY")
                checkAllY()
                resultCount = 0
            }
        }

        var nodeHandLeft = resultBundle.results.first().landmarks().first()[16] //左手腕
        var nodeHandRight = resultBundle.results.first().landmarks().first()[15]  //右手腕

        //当左手的高度高超过鼻子时，自动开始 1 分钟倒数计算，重置所有的数据
        if(nodeHandLeft.y() < nodeNose.y()){
            if(!oneMinuteMode){
                oneMinuteMode = true
                jumpingCount = 0
                oneMinuteJumping.start()
            }
        }
        //当右手的高度
        if(nodeHandRight.y() < nodeNose.y()){
            if(oneMinuteMode){
                oneMinuteMode = false
                jumpingCount = 0
                oneMinuteLeft = ""
                oneMinuteJumping.cancel()
            }
        }

    }
}

// 计算加入到需要计算的点，最后四个点的关系为 4↗3↘2↘1
private fun checkAllY(){
    if(allY.size > 3) {
        var check = true
        for(idx in 0 until allY[0].size){
            var last1 = allY[allY.size-1][idx]
            var last2 = allY[allY.size-2][idx]
            var last3 = allY[allY.size-3][idx]
            var last4 = allY[allY.size-4][idx]
            if (last4 > last3 && last3< last2 && last2 <last1) {
                Log.e(TAG, "DATA Length > 2 check top")
                    check = check && true
            }else{
                check = check && false
            }
        }
        if(check){
            jumpingCount += 1
            Log.e(TAG, "Jumping Count $jumpingCount")
            allY.clear()
        }

    }
}
OverlayView 文件说明
//绘制结果

textPaint.color = Color.YELLOW
textPaint.strokeWidth = LANDMARK_STROKE_WIDTH
textPaint.style = Paint.Style.FILL
textPaint.textSize = 36f

private fun initPaints() {
    //用来画人体框架
    linePaint.color =
        ContextCompat.getColor(context!!, R.color.mp_color_primary)
    linePaint.strokeWidth = LANDMARK_STROKE_WIDTH
    linePaint.style = Paint.Style.STROKE

    //用来画跳绳数量 和 倒数秒数
    textPaint.color = Color.YELLOW
    textPaint.strokeWidth = LANDMARK_STROKE_WIDTH
    textPaint.style = Paint.Style.FILL
    textPaint.textSize = 132f

    //用来显示人体不完整的提示
    hintPaint.color = Color.RED
    hintPaint.strokeWidth = LANDMARK_STROKE_WIDTH
    hintPaint.style = Paint.Style.FILL
    hintPaint.textSize = 96f

    //用来显示关节点
    pointPaint.color = Color.YELLOW
    pointPaint.strokeWidth = LANDMARK_STROKE_WIDTH
    pointPaint.style = Paint.Style.FILL
}
override fun draw(canvas: Canvas) {
    super.draw(canvas)
    results?.let { poseLandmarkerResult ->
        var landmarks = poseLandmarkerResult.landmarks()
        if(jumpingHint.isNotEmpty()){
            canvas.drawText( jumpingHint,100f, 400f, hintPaint)
        }

        if(secondLeft.isNotEmpty()){
            canvas.drawText( secondLeft,300f, 300f, textPaint)
        }
        if (landmarks.size > 0) {
            canvas.drawText( jumpingCount.toString(),100f, 300f, textPaint)
        }

        for(landmark in landmarks) {

            for(normalizedLandmark in landmark) {
                canvas.drawPoint(
                    normalizedLandmark.x() * imageWidth * scaleFactor,
                    normalizedLandmark.y() * imageHeight * scaleFactor,
                    pointPaint
                )
            }

            PoseLandmarker.POSE_LANDMARKS.forEach {
                canvas.drawLine(
                    poseLandmarkerResult.landmarks().get(0).get(it!!.start()).x() * imageWidth * scaleFactor,
                    poseLandmarkerResult.landmarks().get(0).get(it.start()).y() * imageHeight * scaleFactor,
                    poseLandmarkerResult.landmarks().get(0).get(it.end()).x() * imageWidth * scaleFactor,
                    poseLandmarkerResult.landmarks().get(0).get(it.end()).y() * imageHeight * scaleFactor,
                    linePaint)
            }
        }
    }
}

ToDoList