How do I react to a certain amount of time having elapsed since an event, in this webcam reading loop?

Question:

AIM I am making a posture detection system which detects whether a leg is bent or not and by extension counts the Reps done ( a successful rep is when the leg is bent for 8 seconds ). I am doing this by using the media pipe pose model, by calculating the angle between the left ankle, left knee and left hip ( landmarks ).

The aim is to run a countdown timer of 8 seconds as soon the angle becomes less than 140 degrees.

I have tried the following, I am trying to fix this for 2 days, but can’t seem to do it.

import cv2
import numpy as np
import mediapipe as mp
from datetime import timedelta, datetime

mp_drawing = mp.solutions.drawing_utils
mp_pose = mp.solutions.pose



# Capturing/ Reading the video
cap = cv2.VideoCapture('C:/Users/ausaf/Downloads/KneeBendVideo.mp4')

counter = 0
stage = None
diff = 0


def calculate_angle(h, k, a):  # h=hip, k=knee, a=ankle
    h = np.array(h)  # hip
    k = np.array(k)  # knee
    a = np.array(a)  # ankle

    radians = np.arctan2(h[1] - k[1], a[0] - k[0]) - np.arctan2(h[1] - k[1], h[0] - k[0])
    angle = np.abs(radians * 180.0 / np.pi)

    #     if angle > 180:
    #         angle = 360 - angle

    return angle


# # Setup Mediapipe Instance
with mp_pose.Pose(min_detection_confidence=0.8, min_tracking_confidence=0.5) as pose:
    while cap.isOpened():
        ret, frame = cap.read()
        # Recolor Image to RGB
        image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        image.flags.writeable = False

        # Make Detection
        results = pose.process(image)

        # Recolor back to BGR
        image.flags.writeable = True
        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)

        # Extract Landmarks
        try:
            landmarks = results.pose_landmarks.landmark

            # Get Co-ordinates

            hip = [landmarks[mp_pose.PoseLandmark.LEFT_HIP.value].x,  landmarks[mp_pose.PoseLandmark.LEFT_HIP.value].y]
            knee = [landmarks[mp_pose.PoseLandmark.LEFT_KNEE.value].x,
                    landmarks[mp_pose.PoseLandmark.LEFT_KNEE.value].y]
            ankle = [landmarks[mp_pose.PoseLandmark.LEFT_ANKLE.value].x,
                     landmarks[mp_pose.PoseLandmark.LEFT_ANKLE.value].y]

            # Calculate Angle
            angle = calculate_angle(hip, knee, ankle)
            # print(calculate_angle(hip,knee,ankle))

            # visualize
            cv2.putText(image, str(angle),
                        tuple(np.multiply(knee, [854, 640]).astype(int)),
                        cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2, cv2.LINE_AA)
        except:
            pass

        # Render Rep Counter
        # Counter Box
        cv2.rectangle(image, (0, 0), (225, 73), (245, 117, 16), -1)

        # print Rep Counter to window
        cv2.putText(image, 'REPS', (15, 12),
                    cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1, cv2.LINE_AA)
        cv2.putText(image, str(counter), (10, 60),
                    cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 255, 255), 2, cv2.LINE_AA)

        # print Knee Bent Status
        cv2.putText(image, 'Knee Status', (65, 12),
                     cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1, cv2.LINE_AA)
        cv2.putText(image, stage, (60, 60),
                     cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 255, 255), 2, cv2.LINE_AA)

        # Render Detections
        mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS,
                                  mp_drawing.DrawingSpec(color=(245, 117, 66), thickness=2, circle_radius=2),
                                  mp_drawing.DrawingSpec(color=(245, 66, 230), thickness=2, circle_radius=2), )


        # Logic for rep counting **This is the problem**
        if angle > 140:
            stage = 'straight knee'
            
        while angle < 140 and stage == 'straight knee':
            # start timer, take current time
            start_time = datetime.now()
            stage = 'bent knee'
            diff = (datetime.now() - start_time).seconds

            # check if diff == 8
            while diff <= 8 :
                # print diff to putText
                cv2.putText(image, str(diff), (70, 70),
                            cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 0, 0), 2, cv2.LINE_AA)
                diff = (datetime.now() - start_time ).seconds
                print(diff)
            # if the timer hits 8 seconds, increase rep count
            counter = +1
            
        # if the timer is not 0 and the angle becomes greater than 140 degrees (not bent), print feedback to window
        if angle > 140 and diff > 0:
            cv2.putText(image, 'keep your knees bent', (50, 50),
                        cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 0, 0), 2, cv2.LINE_AA)

        cv2.imshow('Mediapipe Feed', image)

        if cv2.waitKey(10) & 0xFF == ord('q'):
            break

cap.release()
cv2.destroyAllWindows()

However, this code does countdown for 8 seconds ( which i know by printing ‘diff’ on the console), but the video output freezes for 8 seconds while its executing. I would appreciate any help.

Asked By: AusafM

||

Answers:

You exit out of this loop only after 8 seconds, and during that time you never show the updated image (and you are not grabbing it from the camera).

while diff <= 8 :
        # print diff to putText
        cv2.putText(image, str(diff), (70, 70),
                    cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 0, 0), 2, cv2.LINE_AA)
        diff = (datetime.now() - start_time ).seconds
        print(diff)

You need to create out of the scope next to the diff start_time=None and then replace both of the while with this:

if angle < 140 and stage == 'straight knee':
    # start timer, take current time
    start_time = datetime.now()
    stage = 'bent knee'

if start_time and stage == 'bent knee':
        diff = (datetime.now() - start_time ).seconds
        if diff <= 8:
            cv2.putText(image, str(diff), (70, 70),
                        cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 0, 0), 2, cv2.LINE_AA)
            print(diff)
Answered By: Ari