Building an AI Workout Form Corrector with React Native and TensorFlow

App
  ├── CameraView                  -- Expo camera with frame callback
  │     └── frameProcessor()      -- Extracts frame for TF.js
  ├── PoseDetector                -- TensorFlow.js MoveNet model
  │     ├── loadModel()           -- Initialize on mount
  │     └── estimatePoses()       -- Returns keypoints per frame
  ├── FormAnalyzer                -- Joint angle calculations
  │     ├── calculateAngle()      -- Vector math for joints
  │     ├── compareForm()         -- Deviation from ideal
  │     └── generateFeedback()    -- Text + severity level
  ├── FeedbackOverlay             -- Visual skeleton + corrections
  │     ├── SkeletonRenderer      -- Draws keypoint connections
  │     └── AlertBadge            -- Shows correction text
  └── AudioFeedback               -- Voice cues via expo-av

npx create-expo-app workout-form-corrector --template blank-typescript
cd workout-form-corrector
npx expo install expo-camera expo-av expo-screen-orientation
npm install @tensorflow/tfjs @tensorflow-models/pose-detection @react-native-async-storage/async-storage
npm install react-native-reanimated react-native-worklets-core

// src/types/pose.ts

export interface Keypoint {
  x: number;
  y: number;
  score: number;
  name: string;
}

export interface Pose {
  keypoints: Keypoint[];
  score: number;
}

export interface JointAngle {
  joint: string;
  angle: number;
  timestamp: number;
}

export type ExerciseType = 'squat' | 'deadlift' | 'overhead_press';

export interface ExerciseProfile {
  name: ExerciseType;
  displayName: string;
  joints: string[];
  idealAngles: Record<string, { min: number; max: number }>;
  phases: ExercisePhase[];
}

export interface ExercisePhase {
  name: string;
  description: string;
  jointConstraints: Record<string, { min: number; max: number }>;
}

export interface FormFeedback {
  joint: string;
  message: string;
  severity: 'info' | 'warning' | 'danger';
  deviation: number;
}

export interface WorkoutSession {
  id: string;
  exercise: ExerciseType;
  startTime: number;
  reps: number;
  feedbackHistory: FormFeedback[];
}

// src/data/exerciseProfiles.ts

import type { ExerciseProfile } from '../types/pose';

export const exerciseProfiles: Record<string, ExerciseProfile> = {
  squat: {
    name: 'squat',
    displayName: 'Squat',
    joints: ['left_knee', 'right_knee', 'left_hip', 'right_hip'],
    idealAngles: {
      left_knee: { min: 70, max: 100 },
      right_knee: { min: 70, max: 100 },
      left_hip: { min: 70, max: 110 },
      right_hip: { min: 70, max: 110 },
    },
    phases: [
      {
        name: 'descent',
        description: 'Lowering phase',
        jointConstraints: {
          left_knee: { min: 90, max: 180 },
          right_knee: { min: 90, max: 180 },
        },
      },
      {
        name: 'bottom',
        description: 'Bottom of squat',
        jointConstraints: {
          left_knee: { min: 70, max: 100 },
          right_knee: { min: 70, max: 100 },
        },
      },
      {
        name: 'ascent',
        description: 'Rising phase',
        jointConstraints: {
          left_knee: { min: 90, max: 180 },
          right_knee: { min: 90, max: 180 },
        },
      },
    ],
  },
  deadlift: {
    name: 'deadlift',
    displayName: 'Deadlift',
    joints: ['left_knee', 'right_knee', 'left_hip', 'right_hip'],
    idealAngles: {
      left_knee: { min: 140, max: 180 },
      right_knee: { min: 140, max: 180 },
      left_hip: { min: 45, max: 90 },
      right_hip: { min: 45, max: 90 },
    },
    phases: [
      {
        name: 'setup',
        description: 'Starting position',
        jointConstraints: {
          left_hip: { min: 45, max: 70 },
          right_hip: { min: 45, max: 70 },
          left_knee: { min: 140, max: 170 },
          right_knee: { min: 140, max: 170 },
        },
      },
      {
        name: 'lift',
        description: 'Lifting phase',
        jointConstraints: {
          left_hip: { min: 70, max: 180 },
          right_hip: { min: 70, max: 180 },
        },
      },
    ],
  },
  overhead_press: {
    name: 'overhead_press',
    displayName: 'Overhead Press',
    joints: ['left_elbow', 'right_elbow', 'left_shoulder', 'right_shoulder'],
    idealAngles: {
      left_elbow: { min: 160, max: 180 },
      right_elbow: { min: 160, max: 180 },
      left_shoulder: { min: 160, max: 180 },
      right_shoulder: { min: 160, max: 180 },
    },
    phases: [
      {
        name: 'rack',
        description: 'Starting position at shoulders',
        jointConstraints: {
          left_elbow: { min: 60, max: 90 },
          right_elbow: { min: 60, max: 90 },
        },
      },
      {
        name: 'press',
        description: 'Pressing overhead',
        jointConstraints: {
          left_elbow: { min: 160, max: 180 },
          right_elbow: { min: 160, max: 180 },
        },
      },
    ],
  },
};

// src/hooks/usePoseDetector.ts

import { useState, useEffect, useRef, useCallback } from 'react';
import * as tf from '@tensorflow/tfjs';
import * as poseDetection from '@tensorflow-models/pose-detection';
import type { Pose, Keypoint } from '../types/pose';

interface UsePoseDetectorReturn {
  poses: Pose[];
  isReady: boolean;
  detectPose: (video: HTMLVideoElement | HTMLCanvasElement) => Promise<void>;
  error: Error | null;
}

export function usePoseDetector(): UsePoseDetectorReturn {
  const [poses, setPoses] = useState<Pose[]>([]);
  const [isReady, setIsReady] = useState(false);
  const [error, setError] = useState<Error | null>(null);
  const detectorRef = useRef<poseDetection.PoseDetector | null>(null);
  const isProcessingRef = useRef(false);

  useEffect(() => {
    async function initializeDetector() {
      try {
        await tf.ready();
        await tf.setBackend('rn-webgl');

        const model = poseDetection.SupportedModels.MoveNet;
        const detectorConfig: poseDetection.MoveNetModelConfig = {
          modelType: poseDetection.movenet.modelType.SINGLEPOSE_LIGHTNING,
          enableSmoothing: true,
        };

        const detector = await poseDetection.createDetector(model, detectorConfig);
        detectorRef.current = detector;
        setIsReady(true);
      } catch (err) {
        setError(err instanceof Error ? err : new Error('Failed to initialize pose detector'));
      }
    }

    initializeDetector();

    return () => {
      detectorRef.current?.dispose();
    };
  }, []);

  const detectPose = useCallback(async (video: HTMLVideoElement | HTMLCanvasElement) => {
    if (!detectorRef.current || isProcessingRef.current) return;

    isProcessingRef.current = true;
    try {
      const estimatedPoses = await detectorRef.current.estimatePoses(video);
      setPoses(
        estimatedPoses.map((pose) => ({
          keypoints: pose.keypoints.map((kp: Keypoint) => ({
            x: kp.x,
            y: kp.y,
            score: kp.score ?? 0,
            name: kp.name,
          })),
          score: pose.score ?? 0,
        }))
      );
    } catch (err) {
      // Silently handle frame processing errors to avoid disrupting the stream
      console.warn('Pose estimation failed for frame:', err);
    } finally {
      isProcessingRef.current = false;
    }
  }, []);

  return { poses, isReady, detectPose, error };
}

// src/analyzer/formAnalyzer.ts

import type { Keypoint, JointAngle, FormFeedback, ExerciseProfile } from '../types/pose';

function toRadians(degrees: number): number {
  return (degrees * Math.PI) / 180;
}

function toDegrees(radians: number): number {
  return (radians * 180) / Math.PI;
}

export function calculateAngle(
  pointA: Keypoint,
  pointB: Keypoint,
  pointC: Keypoint
): number {
  const vectorBA = { x: pointA.x - pointB.x, y: pointA.y - pointB.y };
  const vectorBC = { x: pointC.x - pointB.x, y: pointC.y - pointB.y };

  const dotProduct = vectorBA.x * vectorBC.x + vectorBA.y * vectorBC.y;
  const magnitudeBA = Math.sqrt(vectorBA.x ** 2 + vectorBA.y ** 2);
  const magnitudeBC = Math.sqrt(vectorBC.x ** 2 + vectorBC.y ** 2);

  if (magnitudeBA === 0 || magnitudeBC === 0) return 0;

  const cosineAngle = Math.max(-1, Math.min(1, dotProduct / (magnitudeBA * magnitudeBC)));
  return toDegrees(Math.acos(cosineAngle));
}

// Maps joint names to the three keypoints needed for angle calculation.
// pointB is the vertex of the angle.
const jointKeypointMap: Record<string, [string, string, string]> = {
  left_knee: ['left_hip', 'left_knee', 'left_ankle'],
  right_knee: ['right_hip', 'right_knee', 'right_ankle'],
  left_hip: ['left_shoulder', 'left_hip', 'left_knee'],
  right_hip: ['right_shoulder', 'right_hip', 'right_knee'],
  left_elbow: ['left_shoulder', 'left_elbow', 'left_wrist'],
  right_elbow: ['right_shoulder', 'right_elbow', 'right_wrist'],
  left_shoulder: ['left_hip', 'left_shoulder', 'left_elbow'],
  right_shoulder: ['right_hip', 'right_shoulder', 'right_elbow'],
};

const MIN_KEYPOINT_CONFIDENCE = 0.3;

export function analyzeForm(
  keypoints: Keypoint[],
  profile: ExerciseProfile
): { angles: JointAngle[]; feedback: FormFeedback[] } {
  const keypointMap = new Map<string, Keypoint>();
  for (const kp of keypoints) {
    if (kp.score >= MIN_KEYPOINT_CONFIDENCE) {
      keypointMap.set(kp.name, kp);
    }
  }

  const angles: JointAngle[] = [];
  const feedback: FormFeedback[] = [];
  const now = Date.now();

  for (const joint of profile.joints) {
    const keypointNames = jointKeypointMap[joint];
    if (!keypointNames) continue;

    const [nameA, nameB, nameC] = keypointNames;
    const pointA = keypointMap.get(nameA);
    const pointB = keypointMap.get(nameB);
    const pointC = keypointMap.get(nameC);

    if (!pointA || !pointB || !pointC) continue;

    const angle = calculateAngle(pointA, pointB, pointC);
    angles.push({ joint, angle, timestamp: now });

    const idealRange = profile.idealAngles[joint];
    if (!idealRange) continue;

    if (angle < idealRange.min) {
      const deviation = idealRange.min - angle;
      feedback.push({
        joint,
        message: `Your ${joint.replace(/_/g, ' ')} angle is ${Math.round(angle)} degrees, which is too tight. Try opening up to at least ${idealRange.min} degrees.`,
        severity: deviation > 20 ? 'danger' : 'warning',
        deviation,
      });
    } else if (angle > idealRange.max) {
      const deviation = angle - idealRange.max;
      feedback.push({
        joint,
        message: `Your ${joint.replace(/_/g, ' ')} angle is ${Math.round(angle)} degrees, which is too open. Aim for ${idealRange.max} degrees or less.`,
        severity: deviation > 20 ? 'danger' : 'warning',
        deviation,
      });
    }
  }

  return { angles, feedback };
}

// src/components/SkeletonRenderer.tsx

import React, { useRef, useEffect } from 'react';
import { View, StyleSheet } from 'react-native';
import type { Pose, FormFeedback } from '../types/pose';

interface SkeletonRendererProps {
  poses: Pose[];
  feedback: FormFeedback[];
  width: number;
  height: number;
}

const CONNECTION_PAIRS = [
  ['left_shoulder', 'right_shoulder'],
  ['left_shoulder', 'left_elbow'],
  ['left_elbow', 'left_wrist'],
  ['right_shoulder', 'right_elbow'],
  ['right_elbow', 'right_wrist'],
  ['left_shoulder', 'left_hip'],
  ['right_shoulder', 'right_hip'],
  ['left_hip', 'right_hip'],
  ['left_hip', 'left_knee'],
  ['left_knee', 'left_ankle'],
  ['right_hip', 'right_knee'],
  ['right_knee', 'right_ankle'],
];

const severityColors: Record<string, string> = {
  info: '#3b82f6',
  warning: '#f59e0b',
  danger: '#ef4444',
};

export function SkeletonRenderer({ poses, feedback, width, height }: SkeletonRendererProps) {
  const canvasRef = useRef<HTMLCanvasElement>(null);

  useEffect(() => {
    const canvas = canvasRef.current;
    if (!canvas || poses.length === 0) return;

    const ctx = canvas.getContext('2d');
    if (!ctx) return;

    ctx.clearRect(0, 0, width, height);

    const pose = poses[0];
    const keypointMap = new Map(pose.keypoints.map((kp) => [kp.name, kp]));

    // Create a set of joints that have feedback for color coding
    const feedbackJoints = new Set(feedback.map((f) => f.joint));

    // Draw connections
    ctx.lineWidth = 3;
    for (const [startName, endName] of CONNECTION_PAIRS) {
      const start = keypointMap.get(startName);
      const end = keypointMap.get(endName);
      if (!start || !end) continue;

      const hasIssue = feedbackJoints.has(startName) || feedbackJoints.has(endName);
      ctx.strokeStyle = hasIssue ? '#ef4444' : '#22c55e';
      ctx.beginPath();
      ctx.moveTo(start.x, start.y);
      ctx.lineTo(end.x, end.y);
      ctx.stroke();
    }

    // Draw keypoints
    for (const kp of pose.keypoints) {
      if (kp.score ＜ 0.3) continue;

      const jointFeedback = feedback.find((f) => f.joint === kp.name);
      const color = jointFeedback ? severityColors[jointFeedback.severity] : '#22c55e';

      ctx.beginPath();
      ctx.arc(kp.x, kp.y, 6, 0, 2 * Math.PI);
      ctx.fillStyle = color;
      ctx.fill();
      ctx.strokeStyle = '#ffffff';
      ctx.lineWidth = 2;
      ctx.stroke();
    }
  }, [poses, feedback, width, height]);

  return (
    <View style={[styles.container, { width, height }]}>
      <canvas ref={canvasRef} width={width} height={height} style={styles.canvas} />
    </View>
  );
}

const styles = StyleSheet.create({
  container: {
    position: 'absolute',
    top: 0,
    left: 0,
  },
  canvas: {
    width: '100%',
    height: '100%',
  },
});

// src/screens/WorkoutScreen.tsx

import React, { useState, useRef, useCallback, useEffect } from 'react';
import { View, Text, StyleSheet, TouchableOpacity, Alert } from 'react-native';
import { CameraView } from 'expo-camera';
import { usePoseDetector } from '../hooks/usePoseDetector';
import { analyzeForm } from '../analyzer/formAnalyzer';
import { SkeletonRenderer } from '../components/SkeletonRenderer';
import { exerciseProfiles } from '../data/exerciseProfiles';
import type { ExerciseType, FormFeedback } from '../types/pose';

export function WorkoutScreen() {
  const [selectedExercise, setSelectedExercise] = useState<ExerciseType>('squat');
  const [isWorkoutActive, setIsWorkoutActive] = useState(false);
  const [currentFeedback, setCurrentFeedback] = useState<FormFeedback[]>([]);
  const [dimensions, setDimensions] = useState({ width: 0, height: 0 });
  const { poses, isReady, detectPose, error: detectorError } = usePoseDetector();
  const cameraRef = useRef<CameraView>(null);
  const frameCountRef = useRef(0);

  const profile = exerciseProfiles[selectedExercise];

  useEffect(() => {
    if (!isWorkoutActive || poses.length === 0) return;

    const pose = poses[0];
    const { feedback } = analyzeForm(pose.keypoints, profile);
    setCurrentFeedback(feedback);
  }, [poses, profile, isWorkoutActive]);

  const handleFrame = useCallback(async () => {
    if (!isWorkoutActive || !cameraRef.current) return;

    // Process every other frame to maintain performance
    frameCountRef.current += 1;
    if (frameCountRef.current % 2 !== 0) return;

    try {
      const video = cameraRef.current as unknown as HTMLVideoElement;
      await detectPose(video);
    } catch {
      // Skip frames that fail to process
    }
  }, [isWorkoutActive, detectPose]);

  if (detectorError) {
    return (
      <View style={styles.centerContainer}>
        <Text style={styles.errorText}>Failed to initialize pose detection.</Text>
        <Text style={styles.errorSubtext}>{detectorError.message}</Text>
      </View>
    );
  }

  return (
    <View style={styles.container}>
      <View
        style={styles.cameraContainer}
        onLayout={(event) => {
          const { width, height } = event.nativeEvent.layout;
          setDimensions({ width, height });
        }}
      >
        <CameraView
          ref={cameraRef}
          style={styles.camera}
          facing="front"
          onCameraReady={handleFrame}
        />

        {isWorkoutActive && (
          <SkeletonRenderer
            poses={poses}
            feedback={currentFeedback}
            width={dimensions.width}
            height={dimensions.height}
          />
        )}

        {!isReady && (
          <View style={styles.loadingOverlay}>
            <Text style={styles.loadingText}>Loading AI model...</Text>
          </View>
        )}
      </View>

      {/* Feedback panel */}
      {isWorkoutActive && currentFeedback.length > 0 && (
        <View style={styles.feedbackPanel}>
          {currentFeedback.map((fb, index) => (
            <View
              key={index}
              style={[
                styles.feedbackItem,
                fb.severity === 'danger' && styles.feedbackDanger,
                fb.severity === 'warning' && styles.feedbackWarning,
              ]}
            >
              <Text style={styles.feedbackText}>{fb.message}</Text>
            </View>
          ))}
        </View>
      )}

      {/* Controls */}
      <View style={styles.controls}>
        <View style={styles.exerciseSelector}>
          {(Object.keys(exerciseProfiles) as ExerciseType[]).map((exercise) => (
            <TouchableOpacity
              key={exercise}
              style={[
                styles.exerciseButton,
                selectedExercise === exercise && styles.exerciseButtonActive,
              ]}
              onPress={() => setSelectedExercise(exercise)}
            >
              <Text
                style={[
                  styles.exerciseButtonText,
                  selectedExercise === exercise && styles.exerciseButtonTextActive,
                ]}
              >
                {exerciseProfiles[exercise].displayName}
              </Text>
            </TouchableOpacity>
          ))}
        </View>

        <TouchableOpacity
          style={[styles.startButton, isWorkoutActive ? styles.stopButton : {}]}
          onPress={() => {
            setIsWorkoutActive(!isWorkoutActive);
            setCurrentFeedback([]);
          }}
        >
          <Text style={styles.startButtonText}>
            {isWorkoutActive ? 'Stop Workout' : 'Start Workout'}
          </Text>
        </TouchableOpacity>
      </View>
    </View>
  );
}

const styles = StyleSheet.create({
  container: { flex: 1, backgroundColor: '#000' },
  centerContainer: { flex: 1, justifyContent: 'center', alignItems: 'center', padding: 20 },
  cameraContainer: { flex: 1, position: 'relative' },
  camera: { flex: 1 },
  loadingOverlay: {
    position: 'absolute',
    top: 0,
    left: 0,
    right: 0,
    bottom: 0,
    justifyContent: 'center',
    alignItems: 'center',
    backgroundColor: 'rgba(0, 0, 0, 0.7)',
  },
  loadingText: { color: '#fff', fontSize: 16 },
  feedbackPanel: {
    backgroundColor: 'rgba(0, 0, 0, 0.85)',
    padding: 12,
    maxHeight: 120,
  },
  feedbackItem: {
    padding: 8,
    borderRadius: 6,
    marginBottom: 4,
    backgroundColor: 'rgba(59, 130, 246, 0.2)',
    borderLeftWidth: 3,
    borderLeftColor: '#3b82f6',
  },
  feedbackWarning: {
    backgroundColor: 'rgba(245, 158, 11, 0.2)',
    borderLeftColor: '#f59e0b',
  },
  feedbackDanger: {
    backgroundColor: 'rgba(239, 68, 68, 0.2)',
    borderLeftColor: '#ef4444',
  },
  feedbackText: { color: '#fff', fontSize: 13 },
  controls: { padding: 16, backgroundColor: '#111' },
  exerciseSelector: { flexDirection: 'row', gap: 8, marginBottom: 12 },
  exerciseButton: {
    paddingHorizontal: 14,
    paddingVertical: 8,
    borderRadius: 20,
    backgroundColor: '#222',
    borderWidth: 1,
    borderColor: '#333',
  },
  exerciseButtonActive: { backgroundColor: '#3b82f6', borderColor: '#3b82f6' },
  exerciseButtonText: { color: '#999', fontSize: 13 },
  exerciseButtonTextActive: { color: '#fff' },
  startButton: {
    backgroundColor: '#22c55e',
    paddingVertical: 14,
    borderRadius: 10,
    alignItems: 'center',
  },
  stopButton: { backgroundColor: '#ef4444' },
  startButtonText: { color: '#fff', fontSize: 16, fontWeight: '600' },
  errorText: { fontSize: 16, color: '#ef4444', textAlign: 'center' },
  errorSubtext: { fontSize: 13, color: '#999', marginTop: 8, textAlign: 'center' },
});