DetectHandplay.py

#March 2025
import pygame

import cv2
import numpy as np
import time
import pyautogui
global_param_distance_from_start = 1 # do 1 if you are close to the screen 2 for standing

from enum import Enum
class state(Enum):
    OPEN_HAND = "Open Hand"
    PEACE_SIGN = "Peace Sign"
    THUMB_UP = "Thumb Up"
    THUMB_DOWN = "Thumb Down"
    NONEE = "None"


###############functions

def learn_skin_color(roi):
    """
    Calculate the average color of the skin in the ROI and define HSV boundaries.
    Args:
        roi (numpy.ndarray): The region of interest containing skin pixels.
    Returns:
        lower_skin (numpy.ndarray): Lower HSV boundary for the skin mask.
        upper_skin (numpy.ndarray): Upper HSV boundary for the skin mask.
    """
    hsv = cv2.cvtColor(roi, cv2.COLOR_BGR2HSV)  # Convert ROI to HSV
    mean_color = np.mean(hsv, axis=(0, 1))  # Compute mean HSV value
    std_dev = np.std(hsv, axis=(0, 1))  # Compute standard deviation

    # Define lower and upper bounds based on mean and standard deviation
    lower_skin = np.clip(mean_color, 0, 255).astype(np.uint8)
    upper_skin = np.clip(mean_color, 0, 255).astype(np.uint8)

    # Adjust HSV values with a spread factor for tolerance
    spread_factor = 1.7
    ## 1.7
    lower_skin[2] = max(lower_skin[2] - (std_dev[2] * spread_factor * 1.1), 0)
    upper_skin[2] = min(upper_skin[2] + (std_dev[2] * spread_factor * 1.1), 255)
    lower_skin[1] = max(lower_skin[1] - (std_dev[1] * spread_factor * 0.3), 0)
    upper_skin[1] = min(upper_skin[1] + (std_dev[1] * spread_factor * 0.3), 255)
    lower_skin[0] = max(lower_skin[0] - (std_dev[0] * spread_factor * 0.45), 0)
    upper_skin[0] = min(upper_skin[0] + (std_dev[0] * spread_factor * 0.45), 255)

    return lower_skin, upper_skin


def create_skin_mask(frame, mask, withFill=False, shouldSplitScreen=False, shouldUseRight = False):
    """
    Process the skin mask with morphological operations to clean noise.
    Args:
        frame (numpy.ndarray): The input frame.
        mask (numpy.ndarray): The skin detection mask.
        withFill (bool): Whether to perform extra filling operations.
        shouldSplitScreen (bool): Whether to apply the mask to half of the screen.
    Returns:
        final_mask (numpy.ndarray): The processed skin mask.
    """
    kernel = np.ones((3, 3), np.uint8)
    mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)  # Remove small noise
    mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel)  # Fill small holes
    mask = cv2.medianBlur(mask, 5)
    mask = cv2.erode(mask, kernel, iterations=1)
    mask = cv2.dilate(mask, kernel, iterations=1)
    mask = cv2.GaussianBlur(mask, (7, 7), 0)  # Smooth mask

    if withFill:
        kernel = np.ones((7, 7), np.uint8)
        mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)
        mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel)

    # Further clean edges
    mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel)

    # Define mask for a region (e.g., top-right corner)
    height, width = frame.shape[:2]
    region_mask = np.zeros((height, width), dtype=np.uint8)
    if shouldSplitScreen:
        height //= 2
    if shouldUseRight:
        region_mask[0:height, (3 * width) // 5:width] = 255
    else:
        region_mask[0:height, 0:(2 * width) // 5] = 255

    final_mask = cv2.bitwise_and(mask, region_mask)
    return final_mask


def clean_mask(mask):
    """
    Isolate the largest contour from the given mask.
    Args:
        mask (numpy.ndarray): The binary mask to clean.
    Returns:
        isolated_mask (numpy.ndarray): Mask containing only the largest contour.
    """
    contours, _ = cv2.findContours(mask, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
    largest_contour = max(contours, key=len) if contours else None

    isolated_mask = np.zeros_like(mask, dtype=np.uint8)
    if largest_contour is not None:
        cv2.drawContours(isolated_mask, [largest_contour], -1, 255, thickness=cv2.FILLED)

    return isolated_mask


def is_valid_point(start, far, end):
    """
    Checks if a point is valid based on the blue vector angle (220°-300°).
    :param start: The starting point of the angle
    :param far: The farthest point
    :param end: The ending point of the angle
    :return: True if the point is valid, otherwise False
    """

    def calculate_direction(far, mid):
        """ Calculates the direction of the blue vector in space (0-360°). """
        dx = mid[0] - far[0]
        dy = mid[1] - far[1]
        theta = np.degrees(np.arctan2(-dy, dx))
        if theta < 0:
            theta += 360
        return theta

    # Compute the mid-point of the angle
    mid_x = (start[0] + end[0]) // 2
    mid_y = (start[1] + end[1]) // 2
    direction_angle = calculate_direction(far, (mid_x, mid_y))


    # Check if th



    # e angle is within the invalid range (220°-300°)
    return not (200 <= direction_angle <= 340)

def get_num_of_fingers(contour, frame, isAgodal=False, shouldReturnDStartOfFirstFinger=False):
    """
    Count the number of fingers using convexity defects.
    Args:
        contour (numpy.ndarray): The hand contour.
        frame (numpy.ndarray): The current frame for visualization.
        isAgodal (bool): A flag for additional logic, if we want to find agodal.
        shouldReturnDStartOfFirstFinger (bool): Whether to return the first detected fingertip.
    Returns:
        finger_tips (list): List of fingertip positions.
        isAgodal (bool): Unmodified flag.
    """
    hull = cv2.convexHull(contour, returnPoints=False)
    defects = cv2.convexityDefects(contour, hull)
    # M = cv2.moments(contour)
    # if M["m00"] == 0:
    #     return False, None
    # center_y = int(M["m01"] / M["m00"])
    # counterOfBelowCenter = 0

    if defects is None:
        return False, []

    finger_tips = []
    valid_defect_count = 0

    for i in range(defects.shape[0]):
        s, e, f, d = defects[i, 0]
        start, end, far = tuple(contour[s][0]), tuple(contour[e][0]), tuple(contour[f][0])
        depth = d / 256.0  # Convert depth to float

        # Compute angles to determine valid finger gaps
        a = np.linalg.norm(np.array(start) - np.array(far))
        b = np.linalg.norm(np.array(end) - np.array(far))
        c = np.linalg.norm(np.array(start) - np.array(end))
        angle = np.arccos((a ** 2 + b ** 2 - c ** 2) / (2 * a * b)) * (180 / np.pi)

        depth *= global_param_distance_from_start
        if angle < 80.0 and depth > 10:
            # print(end[1])
            # print(center_y)
            # if center_y > end[1] and counterOfBelowCenter > 0:
            #     print("here2")
            #     continue
            # elif  center_y > end[1]:
            #     print("here1")
            #     counterOfBelowCenter = counterOfBelowCenter +1


            if len(finger_tips) <= 5 and is_valid_point(start, far, end):
                valid_defect_count += 1
                finger_tips.append(start)
                finger_tips.append(end)

                # if shouldReturnDStartOfFirstFinger and len(finger_tips) >= 3:
                #     return far, None

                # Draw defect region
                cv2.line(frame, start, far, (0, 0, 255), 2)
                cv2.line(frame, end, far, (0, 0, 255), 2)
                cv2.circle(frame, far, 5, (0, 255, 0), -1)
                cv2.putText(frame, f"Depth: {int(depth)}", far, cv2.FONT_HERSHEY_SIMPLEX, 0.4, (0, 255, 255), 1)
                cv2.putText(frame, f"Angle: {int(angle)}", (far[0], far[1] + 15), cv2.FONT_HERSHEY_SIMPLEX, 0.4,
                            (0, 255, 255), 1)

    finger_tips = merge_close_points_hand(finger_tips, threshold=30)
    return finger_tips, isAgodal


def verify_hand(frame, fing_tips):
    """
    Verify if the detected contour represents a valid hand based on the number of finger tips detected.
    Args:
        frame: The image frame where the hand is detected.
        fing_tips: List of detected finger tip coordinates.
    Returns:
        is_hand: Boolean indicating if a hand is detected.
        finger_tips: List of valid finger tip coordinates.
    """
    if len(fing_tips) < 3:
        return False, None

    finger_tips = fing_tips
    print(f"Finger count: {len(finger_tips)}")

    for i, tip in enumerate(finger_tips):
        cv2.circle(frame, tip, 10, (255, 0, 0), -1)  # Mark fingertips with blue circles
        cv2.putText(frame, f"Finger {i + 1}", tip, cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1)


    is_hand = len(finger_tips) in {4, 5}  # Consider it a hand if 3-5 fingers are detected
    return is_hand, finger_tips


def detect_peace_sign(contour, finger_tips, isAgudal):
    """
    Detect if a peace sign (two extended fingers) is shown.
    Args:
        contour: Hand contour.
        finger_tips: List of detected finger tip coordinates.
        isAgudal: Boolean indicating if the thumb is extended.
    Returns:
        is_peace: Boolean indicating if a peace sign is detected.
        finger_tips: The detected finger tip positions.
    """
    if isAgudal or len(finger_tips) != 2:
        return False, None

    # Compute palm center to validate finger positions
    M = cv2.moments(contour)
    if M["m00"] == 0:
        return False, None
    center_y = int(M["m01"] / M["m00"])

    for tip in finger_tips:
        if tip[1] >= center_y + 70 / global_param_distance_from_start:
            return False, None

    return True, finger_tips


def detect_thumb_up(contour, frame, number_of_fing, shouldBeUp=True, shouldUseRight = False):
    """
    Detect if a thumb-up gesture is shown.
    Args:
        contour: Hand contour.
        frame: The image frame.
        number_of_fing: Number of detected fingers.
        isAgudal: Boolean indicating if the thumb is extended.
        shouldBeUp: Boolean indicating if the thumb should be pointing upwards.
    Returns:
        is_thumb_up: Boolean indicating if a thumb-up gesture is detected.
        thumb_tip: The detected thumb tip position.
    """
    # if shouldUseRight:
    #     contour = np.flip(contour, axis=0)
    #     contour.reshape(-1, 1, 2)
    hull = cv2.convexHull(contour, returnPoints=False)
    defects = cv2.convexityDefects(contour, hull)
    if defects is None or number_of_fing > 2:
        return False, None

    thumb_tip = None
    valid_defect_count = 0
    potential_tips = []

    # Compute palm center
    M = cv2.moments(contour)
    if M["m00"] == 0:
        return False, None
    center_y = int(M["m01"] / M["m00"])

    for i in range(defects.shape[0]):
        s, e, f, d = defects[i, 0]
        start = tuple(contour[s][0])
        end = tuple(contour[e][0])
        far = tuple(contour[f][0])
        depth = (d / 256.0) * global_param_distance_from_start

        a = np.linalg.norm(np.array(start) - np.array(far))
        b = np.linalg.norm(np.array(end) - np.array(far))
        c = np.linalg.norm(np.array(start) - np.array(end))
        angle = np.arccos((a ** 2 + b ** 2 - c ** 2) / (2 * a * b)) * (180 / np.pi)
        if ( 90 < angle < 180 and 20 < depth < 100):
            if not shouldBeUp and depth > 30 or shouldBeUp:
                potential_tips.append({"start": start, "depth": depth})
                cv2.putText(frame, f"Depth: {int(depth)}", far, cv2.FONT_HERSHEY_SIMPLEX, 0.4, (0, 255, 255), 1)
                cv2.putText(frame, f"Angle: {int(angle)}", (far[0], far[1] + 15), cv2.FONT_HERSHEY_SIMPLEX, 0.4,
                            (0, 255, 255), 1)


    if len(potential_tips) > 1:
        max_tip = max(potential_tips, key=lambda tip: tip["depth"])
        potential_tips = [max_tip]

    if len(potential_tips) == 1:
        thumb_tip = potential_tips[0]["start"]
        if (shouldBeUp and thumb_tip[1] < center_y) or (not shouldBeUp and thumb_tip[1] > center_y):
            valid_defect_count += 1

    if thumb_tip:
        cv2.circle(frame, thumb_tip, 10, (0, 255, 0), -1)

    is_thumb_up = valid_defect_count == 1
    return is_thumb_up, thumb_tip


def merge_close_points_hand(points, threshold=30):
    """
    Merge finger tips that are within a specified distance.
    """
    if not points:
        return []

    merged_points = []
    used = [False] * len(points)

    for i, point in enumerate(points):
        if used[i]:
            continue
        cluster = [point]
        used[i] = True
        for j, other_point in enumerate(points):
            if not used[j] and np.linalg.norm(np.array(point) - np.array(other_point)) < threshold:
                cluster.append(other_point)
                used[j] = True
        avg_point = tuple(np.mean(cluster, axis=0).astype(int))
        merged_points.append(avg_point)

    return merged_points


def startScreen(firstTimeInStart, mask, frame, screen, whereIsDot, timeOfDot, counter,shouldDecideHand):
    game1_rect, game2_rect, screen = startGame(firstTimeInStart, screen, shouldDecideHand)
    isInStart, screen, whichGameToPlay,timeOfDot, counter =getGameInfo(game1_rect, game2_rect, whereIsDot, counter, timeOfDot,  mask, frame, screen)
    return isInStart, screen, whichGameToPlay,timeOfDot, counter


def startGame(firstTimeInStart, screen,shouldDecideHand):
    # Load your images (adjust the paths as needed)
    if shouldDecideHand:
        game1_path = r"LeftHand.jpeg"
        game2_path = r"RightHand.jpeg"
    else:
        game1_path = r"hard.jpeg"
        game2_path = r"easy.jpeg"
    game3_path = r"maya.jpeg"
    game4_path = r"ori.jpeg"
    game5_path = r"dolev.jpeg"
    game6_path = r"Roy.jpeg"
    game1_img = pygame.image.load(game1_path)
    game2_img = pygame.image.load(game2_path)
    game3_img = pygame.image.load(game3_path)
    game4_img = pygame.image.load(game4_path)
    game5_img = pygame.image.load(game5_path)
    game6_img = pygame.image.load(game6_path)
    # Scale the images to make them bigger (e.g., 300×300 each)
    game1_img = pygame.transform.scale(game1_img, (150, 100))
    game2_img = pygame.transform.scale(game2_img, (150, 100))
    game3_img = pygame.transform.scale(game3_img, (250, 250))
    game4_img = pygame.transform.scale(game4_img, (250, 250))
    game5_img = pygame.transform.scale(game5_img, (250, 250))
    game6_img = pygame.transform.scale(game6_img, (250, 250))
    game1_rect = game1_img.get_rect()
    game1_rect.topleft = (20, 150)
    game2_rect = game2_img.get_rect()
    game2_rect.topleft = (300, 150)  # further down
    game3_rect = game3_img.get_rect()
    game3_rect.topleft = (50, 700)
    game4_rect = game4_img.get_rect()
    game4_rect.topleft = (600, 700)
    game5_rect = game5_img.get_rect()
    game5_rect.topleft = (50, 350)
    game6_rect = game6_img.get_rect()
    game6_rect.topleft = (600, 350)
    if firstTimeInStart:
        screen = pygame.display.set_mode((1000, 1000))
        pygame.display.set_caption("Game Selection Screen")

        # Create a bigger font for the main title
        title_font = pygame.font.SysFont(None, 120)  # Larger size
        subtitle_font = pygame.font.SysFont(None, 50)  # Subtitles bigger too

        # Render the main title text
        title_text = title_font.render("Choose Your Game", True, (255, 255, 255))

        # We'll center the title near the top of the screen
        screen_width, screen_height = screen.get_size()
        title_rect = title_text.get_rect(center=(screen_width // 2, 100))

        # Now define positions for the images
        # For example, place them on the left side with some vertical spacing

        # Render subtitles for each image
        if shouldDecideHand:
            text = "Left Hand"
        else:
            text = "Brave"
        game1_subtitle = subtitle_font.render(text, True, (255, 255, 255))
        game1_subtitle_rect = game1_subtitle.get_rect(midtop=(game1_rect.centerx, game1_rect.bottom + 10))
        if shouldDecideHand:
            text2 = "Right Hand"
        else:
            text2 = "Cowardly"

        game2_subtitle = subtitle_font.render(text2, True, (255, 255, 255))
        game2_subtitle_rect = game2_subtitle.get_rect(midtop=(game2_rect.centerx, game2_rect.bottom + 10))

        game3_subtitle = subtitle_font.render("Maya", True, (255, 255, 255))
        game3_subtitle_rect = game3_subtitle.get_rect(midtop=(game3_rect.centerx, game3_rect.bottom + 10))

        game4_subtitle = subtitle_font.render("Ori", True, (255, 255, 255))
        game4_subtitle_rect = game4_subtitle.get_rect(midtop=(game4_rect.centerx, game4_rect.bottom + 10))

        game5_subtitle = subtitle_font.render("Dolev", True, (255, 255, 255))
        game5_subtitle_rect = game5_subtitle.get_rect(midtop=(game5_rect.centerx, game5_rect.bottom + 10))

        game6_subtitle = subtitle_font.render("Roy", True, (255, 255, 255))
        game6_subtitle_rect = game6_subtitle.get_rect(midtop=(game6_rect.centerx, game6_rect.bottom + 10))

        # Fill the background
        screen.fill((50, 50, 100))

        # Draw the title
        screen.blit(title_text, title_rect)

        # Draw the images and their subtitles
        screen.blit(game1_img, game1_rect)
        screen.blit(game1_subtitle, game1_subtitle_rect)

        screen.blit(game2_img, game2_rect)
        screen.blit(game2_subtitle, game2_subtitle_rect)

        # Draw the images and their subtitles
        screen.blit(game3_img, game3_rect)
        screen.blit(game3_subtitle, game3_subtitle_rect)

        screen.blit(game4_img, game4_rect)
        screen.blit(game4_subtitle, game4_subtitle_rect)  # Draw the images and their subtitles

        screen.blit(game5_img, game5_rect)
        screen.blit(game5_subtitle, game5_subtitle_rect)

        screen.blit(game6_img, game6_rect)
        screen.blit(game6_subtitle, game6_subtitle_rect)

        pygame.display.flip()
    return game1_rect, game2_rect, screen

def getGameInfo(game1_rect, game2_rect, whereIsDot, counter, timeOfDot,  mask, frame, screen):
    # Draw the "start" point as a small red dot
    contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

    if contours:
        # Find the largest contour
        hand_contour = max(contours, key=cv2.contourArea)
        contour_area = cv2.contourArea(hand_contour)
        contour_area = contour_area*global_param_distance_from_start
        if (5000 < contour_area < 200000) :  # Minimum and maximum size thresholds
            # fing_start, _ = get_num_of_fingers(hand_contour, frame, False, True)
            # if (len(fing_start) > 2): #todo try to remove this to 0
            #     fing_start = sorted(fing_start, key=lambda p: p[1])
            #     fing_start = fing_start[0]
            # Compute the convex hull
            hull = cv2.convexHull(hand_contour, returnPoints=False)
            defects = cv2.convexityDefects(hand_contour, hull)

            if defects is not None:
                fingertips = []
                for i in range(defects.shape[0]):
                    s, e, f, d = defects[i, 0]
                    start = tuple(hand_contour[s][0])
                    end = tuple(hand_contour[e][0])
                    far = tuple(hand_contour[f][0])

                    # Fingertips are the start and end points of convexity defects
                    fingertips.append(start)
                    fingertips.append(end)

                # Sort fingertips by height (y-coordinate, lowest value is highest)
                fingertips = sorted(fingertips, key=lambda p: p[1])

                if fingertips:
                    # Use the highest detected fingertip
                    cx, cy = fingertips[0]
                    fing_start = fingertips

                    # Draw fingertip marker
                    cv2.circle(frame, (cx, cy), 10, (0, 0, 255), -1)

                    # Save the pixels behind the circle before drawing
                    circle_rect = pygame.Rect(cx - 10, cy - 10, 20, 20)
                    if screen.get_rect().contains(circle_rect):
                        saved_pixels = screen.subsurface(circle_rect).copy()
                    else:
                        saved_pixels = None

                    # Draw the new circle
                    pygame.draw.circle(screen, (255, 0, 0), (cx, cy), 10)
                    pygame.display.flip()  # Update the screen
                    if game2_rect.collidepoint((cx, cy)):
                        if not whereIsDot == 'easy':
                            counter = 1
                            print('in easy once')
                            timeOfDot= time.time()
                            whereIsDot = 'easy'
                        elif time.time() - timeOfDot > 4 and counter > 1:
                            print("we are playing the game:")
                            return  False, screen, whereIsDot, timeOfDot, counter
                        counter = counter +1
                    elif game1_rect.collidepoint((cx, cy)):
                        print('hard')
                        if not whereIsDot == 'hard':
                            counter = 1
                            timeOfDot= time.time()
                            whereIsDot = 'hard'
                        elif time.time() - timeOfDot > 4 and counter > 5:
                            print("we are playing the game:")
                            return  False, screen, whereIsDot, timeOfDot, counter
                        counter = counter + 1
                    else:
                        print('in else')
                        timeOfDot = time.time()
                    print(time.time() - timeOfDot)

                    # Wait for 50 milisecond
                    pygame.time.delay(50)

                    # Restore the pixels that were behind the circle
                    if saved_pixels is not None:
                        screen.blit(saved_pixels, circle_rect.topleft)
                    pygame.display.flip()  # Update the screen again
    return True, screen,whereIsDot, timeOfDot, counter


def instructionsForHardGame():
    pygame.init()
    # Set up the display (e.g., 1200x800; you can also use full screen)
    screen_width, screen_height = 1200, 800
    screen = pygame.display.set_mode((screen_width, screen_height))
    pygame.display.set_caption("Game Title Page")

    # Load your background image and scale it to fill the screen
    bg_image_path = r"background.jpeg"  # update this path
    bg_image = pygame.image.load(bg_image_path)
    bg_image = pygame.transform.scale(bg_image, (screen_width, screen_height))

    # Scale the image to fill the screen
    bg_image = pygame.transform.scale(bg_image, (screen_width, screen_height))

    # Load your images (update the file paths accordingly)
    image_paths = [
        r"open.jpg",
        r"left.jpg",
        r"up.jpg",
        r"down.jpg"
    ]

    images = []
    for path in image_paths:
        img = pygame.image.load(path)
        images.append(img)

    # Optionally scale the images to a desired size
    scaled_size = (250, 250)
    images = [pygame.transform.scale(img, scaled_size) for img in images]

    # Define captions for each image (you can change these as needed)
    captions = [
        "RIGHT",
        "LEFT",
        "UP",
        "DOWN"
    ]

    # Create fonts for title and captions
    title_font = pygame.font.SysFont(None, 80)  # Big title font
    caption_font = pygame.font.SysFont(None, 40)  # Caption font

    # Render the title text
    title_text = title_font.render("instructions - read carefully", True, (255, 255, 255))
    title_rect = title_text.get_rect(center=(screen_width // 2, 50))

    # Row 1: Image 1 at (50, 150), Image 2 at (50, 400)
    # Row 2: Image 3 at (50, 650), Image 4 at (50, 900) -- note: adjust vertical spacing as needed
    # Since our window is 1200x800, we use two rows.
    positions = [
        (50, 150),  # Image 1 position (top left)
        (50, 450),  # Image 2 position (bottom left)
        (900, 150),  # Image 3 position (top right)
        (900, 450)  # Image 4 position (bottom right)
    ]

    start_time = time.time()

    while time.time() - start_time <4 :
        # Fill the background with a color (e.g., dark blue)
        screen.fill((30, 30, 60))

        # Draw the transparent background image
        screen.blit(bg_image, (0, 0))

        # Draw the title at the top (centered)
        screen.blit(title_text, title_rect)

        # Draw the images and their captions
        for i, pos in enumerate(positions):
            # Draw the image
            screen.blit(images[i], pos)
            # Render caption text
            caption_text = caption_font.render(captions[i], True, (255, 255, 255))
            # Place caption to the right of the image with a small offset
            caption_rect = caption_text.get_rect(midleft=(pos[0] + scaled_size[0] // 3 + 10, pos[1] - 20))
            screen.blit(caption_text, caption_rect)
        # Calculate the time left (start_time - 4)
        elapsed_time = time.time() - start_time
        timer_value = 4 - elapsed_time # You can adjust this as needed

        # Create a timer text to show on the screen
        font = pygame.font.SysFont(None, 48)
        timer_text = font.render(f"Time: {max(timer_value - elapsed_time, 0):.2f}", True, (255, 255, 255))

        # Draw the timer on the screen at a specific position (e.g., top left corner)
        screen.blit(timer_text, (10, 10))

        # Update the display
        pygame.display.flip()

        pygame.display.flip()


    pygame.quit()


# Global variable to store the previous fingertip position
prev_fingertip_position = None

# Smoothing factor for the exponential moving average (EMA)
alpha = 1


def track_hand_easy(frame, mask, prev_detect, shouldPlay, shouldUseRight = False):
    """
    Track the hand by finding the largest contour and determining which region of the screen it's in.
    Args:
        frame: The full frame (numpy array).
        mask: The binary mask for detected hand regions.
        prev_hand_position: Previous position of the hand (tuple).
    Returns:
        frame: Frame with tracking visualizations.
        current_hand_position: Current position of the hand (tuple).
        detected_region: The region where the hand is located.
    """
    global prev_fingertip_position  # Allow smoothing across function calls

    contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    detected_region = "Unknown"  # Default region

    if contours:
        # Find the largest contour (assumed to be the hand)
        hand_contour = max(contours, key=cv2.contourArea)

        # Filter by contour area to remove noise
        contour_area = cv2.contourArea(hand_contour)
        if 5000 < contour_area < 200000:
            # Compute the convex hull
            hull = cv2.convexHull(hand_contour, returnPoints=False)
            defects = cv2.convexityDefects(hand_contour, hull)

            if defects is not None:
                fingertips = []
                for i in range(defects.shape[0]):
                    s, e, f, d = defects[i, 0]
                    start = tuple(hand_contour[s][0])
                    end = tuple(hand_contour[e][0])
                    far = tuple(hand_contour[f][0])

                    # Fingertips are the start and end points of convexity defects
                    fingertips.append(start)
                    fingertips.append(end)

                # Sort fingertips by height (y-coordinate, lowest value is highest)
                fingertips = sorted(fingertips, key=lambda p: p[1])

                if fingertips:
                    # Use the highest detected fingertip
                    cx, cy = fingertips[0]
                    prev_fingertip_position = (cx, cy)

                    # Draw fingertip marker
                    cv2.circle(frame, (cx, cy), 10, (0, 0, 255), -1)

                    # Define screen regions for interaction
                    height, width = frame.shape[:2]
                    left_half_width = int(height * 2 / 3)  # The width of the left side
                    upper_two_thirds = int(height * 2 / 3)  # Upper 2/3 height
                    if shouldUseRight:
                        regions = {
                            "Up": (width - left_half_width, 0, width, int(upper_two_thirds * 1 / 3)),
                            "Down": (width - left_half_width, int(upper_two_thirds * 2 / 3), width, upper_two_thirds),
                            "Left": (
                                width - left_half_width, int(upper_two_thirds * 1 / 3),
                                width - (left_half_width // 3) * 2,
                                int(upper_two_thirds * 2 / 3)),
                            "Center": (
                                width - (left_half_width // 3) * 2, int(upper_two_thirds * 1 / 3),
                                width - left_half_width // 3,
                                int(upper_two_thirds * 2 / 3)),
                            "Right": (width - left_half_width // 3, int(upper_two_thirds * 1 / 3), width,
                                      int(upper_two_thirds * 2 / 3))
                        }
                    else:
                        regions = {
                            "Up": (0, 0, left_half_width, int(upper_two_thirds * 1 / 3)),
                            "Down": (0, int(upper_two_thirds * 2 / 3), left_half_width, upper_two_thirds),
                            "Left": (
                            0, int(upper_two_thirds * 1 / 3), left_half_width // 3, int(upper_two_thirds * 2 / 3)),
                            "Center": (
                                left_half_width // 3, int(upper_two_thirds * 1 / 3), (left_half_width // 3) * 2,
                                int(upper_two_thirds * 2 / 3)),
                            "Right": (
                                (left_half_width // 3) * 2, int(upper_two_thirds * 1 / 3), left_half_width,
                                int(upper_two_thirds * 2 / 3))
                        }

                    # Check which region the fingertip is in
                    detected_region = "Unknown"
                    for region_name, (x1, y1, x2, y2) in regions.items():
                        if x1 <= cx <= x2 and y1 <= cy <= y2:
                            detected_region = region_name
                            break

                    if detected_region != 'Center' and prev_detect != detected_region:
                        # Display detected region on the frame
                        cv2.putText(frame, f"Fingertip in: {detected_region}", (10, height - 20),
                                    cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2)
                        if shouldPlay:
                            pyautogui.press(detected_region)

    # Redefine screen regions (in case needed for another purpose)
    height, width = frame.shape[:2]
    left_half_width = int(height * 2 / 3)
    upper_two_thirds = int(height * 2 / 3)
    if shouldUseRight:
        regions = {
            "Up": (width - left_half_width, 0, width, int(upper_two_thirds * 1 / 3)),
            "Down": (width - left_half_width, int(upper_two_thirds * 2 / 3), width, upper_two_thirds),
            "Left": (
                width - left_half_width, int(upper_two_thirds * 1 / 3), width - (left_half_width // 3) * 2,
                int(upper_two_thirds * 2 / 3)),
            "Center": (
                width - (left_half_width // 3) * 2, int(upper_two_thirds * 1 / 3), width - left_half_width // 3,
                int(upper_two_thirds * 2 / 3)),
            "Right": (width - left_half_width // 3, int(upper_two_thirds * 1 / 3), width,
                      int(upper_two_thirds * 2 / 3))
        }
    else:
        regions = {
            "Up": (0, 0, left_half_width, int(upper_two_thirds * 1 / 3)),
            "Down": (0, int(upper_two_thirds * 2 / 3), left_half_width, upper_two_thirds),
            "Left": (0, int(upper_two_thirds * 1 / 3), left_half_width // 3, int(upper_two_thirds * 2 / 3)),
            "Center": (
            left_half_width // 3, int(upper_two_thirds * 1 / 3), (left_half_width // 3) * 2, int(upper_two_thirds * 2 / 3)),
            "Right": (
            (left_half_width // 3) * 2, int(upper_two_thirds * 1 / 3), left_half_width, int(upper_two_thirds * 2 / 3))
        }

    # Draw the regions on the frame
    for name, (x1, y1, x2, y2) in regions.items():
        color = (255, 255, 255) if name != "Center" else (0, 255, 0)  # Center in green
        cv2.rectangle(frame, (x1, y1), (x2, y2), color, 2)
        cv2.putText(frame, name, (x1 + 10, y1 + 20), cv2.FONT_HERSHEY_SIMPLEX, 0.6, color, 2)

    cv2.namedWindow("Hand Detection and Tracking", cv2.WINDOW_NORMAL)
    cv2.resizeWindow("Hand Detection and Tracking", 1280 + 300, 720 + 300)
    cv2.imshow("Hand Detection and Tracking", frame)

    return frame, prev_fingertip_position, detected_region


def track_hand(frame, mask, shouldPlay, prev_hand_position, prevState=state.NONEE, shouldUseRight = False):
    """
    Track the hand by finding the largest contour and its center.
    Args:
        frame: The full frame (numpy array).
        mask: The binary mask for skin regions.
        prev_hand_position: Previous position of the hand (tuple).
    Returns:
        frame: Frame with hand tracking visualizations.
        current_hand_position: Current position of the hand (tuple).
    """
    contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    hand_contour = None
    timeToSleep = 0.03
    thisState = state.NONEE

    if contours:
        # Find the largest contour (assumed to be the hand)
        hand_contour = max(contours, key=cv2.contourArea)
        contour_area = cv2.contourArea(hand_contour)
        contour_area = contour_area * global_param_distance_from_start

        if 5000 < contour_area < 200000:  # Filter by size threshold
            # Calculate the center of the hand
            if shouldUseRight:
                hand_contour = np.flip(hand_contour, axis=0)
                hand_contour.reshape(-1, 1, 2)


            fing_tips, isAgudal = get_num_of_fingers(hand_contour, frame)
            if isinstance(fing_tips, bool):
                fing_tips = []

            num_of_fing_tips = len(fing_tips)
            print(f"Finger length: {num_of_fing_tips}")

            is_hand, finger_tips = verify_hand(frame, fing_tips)
            thisState = state.OPEN_HAND if is_hand else thisState

            if is_hand:
                if thisState == prevState:
                    if shouldPlay:
                        pyautogui.press('right')
                        time.sleep(timeToSleep)
                    cv2.putText(frame, "Open hand Gesture Detected!", (50, 50),
                                cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2)
            else:
                # Check for specific hand gestures
                is_hand_index, finger_tips = detect_peace_sign(hand_contour, fing_tips, isAgudal)
                thisState = state.PEACE_SIGN if is_hand_index else thisState
                if is_hand_index:
                    if thisState == prevState:
                        if shouldPlay:
                            pyautogui.press('left')
                            time.sleep(timeToSleep)
                        cv2.putText(frame, "Peace sign Detected!", (50, 50),
                                    cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2)

                else:
                    # Check for thumbs up gesture
                    is_hand_index, finger_tips = detect_thumb_up(hand_contour, frame, num_of_fing_tips, True, shouldUseRight)
                    thisState = state.THUMB_UP if is_hand_index else thisState
                    if is_hand_index:
                        if thisState == prevState:
                            if shouldPlay:
                                pyautogui.press('up')
                                time.sleep(timeToSleep)
                            cv2.putText(frame, "Thumb up", (50, 50),
                                        cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2)
                    else:
                        # Check for thumbs down gesture
                        is_hand_index, finger_tips = detect_thumb_up(hand_contour, frame, num_of_fing_tips, False, shouldUseRight)
                        thisState = state.THUMB_DOWN if is_hand_index else thisState
                        if is_hand_index:
                            if thisState == prevState:
                                if shouldPlay:
                                    pyautogui.press('down')
                                    time.sleep(timeToSleep)
                                cv2.putText(frame, "Thumb down", (50, 50),
                                            cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2)
            if (is_hand):
                print("is hand")
                M = cv2.moments(hand_contour)
                if M["m00"] > 0:
                    cx = int(M["m10"] / M["m00"])
                    cy = int(M["m01"] / M["m00"])
                    current_hand_position = (cx, cy)

                    # Draw the contour and center of the hand
                    cv2.drawContours(frame, [hand_contour], -1, (0, 255, 0), 5)
                    cv2.circle(frame, current_hand_position, 8, (0, 0, 255), -1)

                    return frame, current_hand_position, thisState

    return frame, prev_hand_position, thisState
def switchScreen(shouldSwitchToGame, shouldPlay):
    if shouldSwitchToGame and shouldPlay:
        print("this is how I am")
        # Press and hold the "command" key
        pyautogui.keyDown('command')

        # Press and release the "tab" key
        pyautogui.press('tab')
        time.sleep(1)  # todo remove this sleep and change left or right as needed
        pyautogui.press('right')
        pyautogui.keyUp('command')
        pyautogui.press('enter')
        return False
#######################################################

# Main program
cap = cv2.VideoCapture(0)
roi_coords = (105, 115, 70, 70)  # Define the ROI (x, y, width, height)
start_time = time.time()
lower_skin, upper_skin = None, None
prev_hand_position = None

frame_count = 0
prevState = state.NONEE
isInStart = True
isInStartFirst = True
firstTimeInStart = True
screen = None
whichGameToPlay = ''
whichGame = ''
timeOfDot = 0
pygame.init()
shouldSwitchToGame = True
shouldPlay = True
prev_detect = "Unknown"
counter = 0
mask_window_name = ""
mask = None
shouldUseRight = False
notInStartFirstTime = True
isInStartNew = True


def timerDisplay(frame, timer_sec, elapsed_time):
    # Display the timer
    time_remaining = timer_sec - int(elapsed_time)
    timer_text = f"Time remaining: {time_remaining} sec"
    cv2.putText(frame, timer_text, (10, 60), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 0, 0), 2)
def textDisplay(frame, text):
    cv2.putText(frame, text, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 255), 2)


while True:
    ret, frame = cap.read()
    if not ret:
        for i in range(50):
            ret, frame = cap.read()
            if ret:
                break
        if not ret:
            print("ret")
            if isInStart:
                firstTimeInStart = True
            continue

    frame = cv2.flip(frame, 1)  # Flip the frame for better user experience
    elapsed_time = time.time() - start_time

    x, y, w, h = roi_coords
    roi = frame[y:y + h, x:x + w]  # Extract the ROI
    timer_sec = 10

    if elapsed_time <= timer_sec:
         # Display the green rectangle for 5 seconds
        cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)
        text = "Place your hand in the green box to learn skin color..."
        textDisplay(frame, text)
        timerDisplay(frame, timer_sec, elapsed_time)
         # Learn skin color at the last second of 5 seconds
        if elapsed_time > timer_sec - 1:
            lower_skin1, upper_skin1 = learn_skin_color(roi)
            print("Skin color boundaries set!")
            print(f"Lower HSV: {lower_skin}")
            print(f"Upper HSV: {upper_skin}")
    elif elapsed_time <= timer_sec +5:
        text = "Give your front hand"
        textDisplay(frame, text)
        timerDisplay(frame, timer_sec + 5, elapsed_time)
        # Create an empty mask of the same size as the input image
        hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)  # Convert frame to HSV
        mask = cv2.inRange(hsv, lower_skin1 , upper_skin1 )  # Create the mask

        mask = create_skin_mask(frame, mask, True, True)
        contour_mask = np.zeros_like(mask, dtype=np.uint8)
        contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

        if contours:
            # Step 2: Find the largest contour (assumed to be the hand)
            hand_contour = max(contours, key=cv2.contourArea)
            M = cv2.moments(hand_contour)
            if M["m00"] > 0:
                cx = int(M["m10"] / M["m00"])
                cy = int(M["m01"] / M["m00"])
                current_hand_position = (cx, cy)

                # Draw the contour and center of the hand
                cv2.drawContours(frame, [hand_contour], -1, (0, 0, 255), 5)
                cv2.circle(frame, current_hand_position, 8, (0, 0, 255), -1)

            # Step 3: Create a mask for the largest contour
            contour_mask = np.zeros_like(mask, dtype=np.uint8)
            cv2.drawContours(contour_mask, [hand_contour], -1, 255, thickness=cv2.FILLED)

            # Step 4: Apply the mask to the original frame and convert to HSV
            hsv_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
            hsv_masked = hsv_frame[contour_mask == 255]  # Get HSV values inside the contour mask

            # Step 5: Calculate mean and standard deviation for non-zero HSV pixels
            mean_color = np.mean(hsv_masked, axis=0)
            std_dev = np.std(hsv_masked, axis=0)

            # Step 6: Define dynamic skin color bounds
            lower_skin = np.clip(mean_color - (std_dev * 1.2), 0, 255).astype(np.uint8)
            upper_skin = np.clip(mean_color + (std_dev * 1.2), 0, 255).astype(np.uint8)
    elif elapsed_time <= timer_sec +10:
        text = "Give your back hand"
        textDisplay(frame, text)
        timerDisplay(frame, timer_sec + 10, elapsed_time)
        # Create an empty mask of the same size as the input image
        hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)  # Convert frame to HSV
        mask = cv2.inRange(hsv, lower_skin1, upper_skin1)  # Create the mask
        mask = create_skin_mask(frame, mask, True, True)
        contour_mask = np.zeros_like(mask, dtype=np.uint8)
        contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

        if contours:
            # Step 2: Find the largest contour (assumed to be the hand)
            hand_contour = max(contours, key=cv2.contourArea)
            M = cv2.moments(hand_contour)
            if M["m00"] > 0:
                cx = int(M["m10"] / M["m00"])
                cy = int(M["m01"] / M["m00"])
                current_hand_position = (cx, cy)

                # Draw the contour and center of the hand
                cv2.drawContours(frame, [hand_contour], -1, (0, 0, 255), 5)
                cv2.circle(frame, current_hand_position, 8, (0, 0, 255), -1)

            # Step 3: Create a mask for the largest contour
            contour_mask = np.zeros_like(mask, dtype=np.uint8)
            cv2.drawContours(contour_mask, [hand_contour], -1, 255, thickness=cv2.FILLED)

            # Step 4: Apply the mask to the original frame and convert to HSV
            hsv_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
            hsv_masked = hsv_frame[contour_mask == 255]  # Get HSV values inside the contour mask

            # Step 5: Calculate mean and standard deviation for non-zero HSV pixels
            mean_color = np.mean(hsv_masked, axis=0)
            std_dev = np.std(hsv_masked, axis=0)

            # Step 6: Define dynamic skin color bounds
            lower_skin2 = np.clip(mean_color - (std_dev * 1.5), 0, 255).astype(np.uint8)
            upper_skin2 = np.clip(mean_color + (std_dev * 1.5), 0, 255).astype(np.uint8)
            print(lower_skin,upper_skin,lower_skin2, upper_skin2)
            print(type(lower_skin2))
    else:


        if lower_skin is not None and upper_skin is not None:
            hsv_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
            mask1 = cv2.inRange(hsv_frame, lower_skin, upper_skin)
            mask2 = cv2.inRange(hsv_frame, lower_skin2, upper_skin2)
            combined_mask = cv2.bitwise_or(mask1, mask2)
            if isInStart or isInStartNew:
                shouldUseRight = False
            if whichGameToPlay == 'hard' and not isInStart or isInStart or isInStartNew:
                mask = create_skin_mask(frame, combined_mask, True, True, shouldUseRight)
            else:
                mask = create_skin_mask(frame, combined_mask,True, False, shouldUseRight)
            if mask.any() != 0:
                mask = clean_mask(mask)
            if frame_count % 50 ==0 and shouldPlay:
                pyautogui.press('enter') #this is to restart the game

            # Perform hand tracking every 3 frames
            if frame_count % 2 == 0:
                if isInStart:
                    isInStart, screen, whichGameToPlay,timeOfDot, counter=  startScreen(firstTimeInStart,mask, frame, screen,whichGameToPlay, timeOfDot, counter, False)
                    firstTimeInStart = False
                elif isInStartNew:
                    if notInStartFirstTime:
                        screen = None
                        timeOfDot = 0
                    isInStartNew, screen, whichGame, timeOfDot, counter = startScreen(notInStartFirstTime, mask,
                                                                                         frame, screen,
                                                                                         whichGame, timeOfDot,
                                                                                         counter, True)
                    notInStartFirstTime = False
                    if not isInStartNew  :
                        if whichGame == 'easy':
                            shouldUseRight = True
                        else:
                            whichGame = False
                        if whichGameToPlay == 'hard':
                            instructionsForHardGame()
                        pygame.display.quit()
                        pygame.quit()

                elif whichGameToPlay == 'hard':
                    shouldSwitchToGame = switchScreen(shouldSwitchToGame, shouldPlay)
                    frame, prev_hand_position, prevState = track_hand(frame, mask,shouldPlay, prev_hand_position, prevState, shouldUseRight)
                else:
                    shouldSwitchToGame = switchScreen(shouldSwitchToGame, shouldPlay)
                    frame, prev_hand_position, detected_region = track_hand_easy(frame, mask,
                                                                            prev_detect, shouldPlay, shouldUseRight)
                    prev_detect = detected_region

    mask_window_name = "Skin Mask"
    cv2.namedWindow("Skin Mask", cv2.WINDOW_NORMAL)  # For a larger window
    height, width = frame.shape[:2]
    cv2.resizeWindow("Skin Mask", (7 * width) // 8, (7 * height) // 8)  # Adjust size
    if mask  is not None:
        cv2.imshow("Skin Mask", mask)
    else:
        cv2.imshow("Skin Mask", frame)
    # Display the main frame
    window_name = "Hand Detection and Tracking"
    cv2.namedWindow(window_name, cv2.WINDOW_NORMAL)
    height, width = frame.shape[:2]
    if not isInStartNew and not isInStart:#todo move this back
        cv2.resizeWindow("Hand Detection and Tracking", (4*width)//8,(4*height)//8)
        cv2.moveWindow(window_name, 1850, 600)
        cv2.setWindowProperty(window_name, cv2.WND_PROP_TOPMOST, 1)
        cv2.setWindowProperty(mask_window_name, cv2.WND_PROP_TOPMOST, 1)
    else:
        cv2.resizeWindow("Hand Detection and Tracking", (7 * width) // 8, (7 * height) // 8)
        cv2.moveWindow(window_name, 1000, 0)

    cv2.imshow("Hand Detection and Tracking", frame)

    frame_count += 1

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

# Global variable to hold the latest hand center (or contour)
hand_center = None
tracking_running = True  # flag for our tracking thread
cap.release()
cv2.destroyAllWindows()