Portal Site / image_recognition Site

MediaPipe Class for iPBL

• <mediapipe 0.10.33.0> or later
• MediaPipe Examples

---

specification of mediapipe and our mediapipe class

• PoseLandmark
• Hands
  • HandLandmark
  • HandGestureRecognition
• FaceLandmark
• FaceDetection
• ObjectDetecion
• ImageSegmentation

	Pose	Hand	HandGes	Face	FaceDtc	Obj	Seg
image to use	frame	frame	frame	flipped_frame	flipped_frame	either	either
num_detected_*	o	o	o	o	o	o	-
num_landmarks	o	o	o	o	o	-	-
coordinate data	get_landmark	get_landmark	get_landmark	get_landmark	get_landmark get_bounding_box	get_bounding_box	-
optional	segmentation_mask	handedness	handedness gesture_name	-	-	category_name	segmentation_mask confidence_mask

---

🟥 Learning Tasks

1. The following how to use is a simple description of use of our class. Let's try to run them.
2. Next, go to the following page, which outlines the detailed usage of each of them, and execute them all.
   1. ⭕HOW2USE(more) ⬅️ Next

---

🟥 how to use

• The following is a simple description of use of our class.
  • ❗ Note that these programs must be placed in the same directory as our Mediapipe Class file to work.
    • MediaPipeClass.zip
  • Create a new python file in the same directory as our Mediapipe Class file, copy and paste the following sample code, and run it.

⭕ HandLandmark

• draw hand landmarks and handedness, and print the number of detected hands
  

# myhand_simple.py
import os
os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0"
import cv2
from MediapipeHandLandmark import MediapipeHandLandmark as HandLmk

cap = cv2.VideoCapture(0)
Hand = HandLmk(mode="video")
while cap.isOpened():
    ret, frame = cap.read()
    Hand.detect(frame)
    print(Hand.num_detected_hands)
    annotated_frame = Hand.visualize(frame)
    cv2.imshow('annotated frame', annotated_frame)
    key = cv2.waitKey(1)&0xFF
    if key == ord('q'):
        break
cv2.destroyAllWindows()
Hand.release()
cap.release()

# myhand_simple_image.py
import cv2
from MediapipeHandLandmark import MediapipeHandLandmark as HandLmk

img = cv2.imread("./img/standard/Balloon.bmp")
Hand = HandLmk(mode="image")
results = Hand.detect(img)
annotated_img = Hand.visualize(img)
cv2.imshow("annotated_img", annotated_img)
cv2.waitKey(0)
Hand.release()

⭕ HandGestureRecognition

• draw hand landmarks and handedness, and print gesture name and its score
  • recognizable gesture: None, Closed_Fist, Open_Plam, Pointing_up, Thumb_Down, Thumb_Up, Victory, ILoveYou
    
  

# myhand_ges_simple.py
import os
os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0"
import cv2
from MediapipeHandGestureRecognition import MediapipeHandGestureRecognition as HandGesRec

cap = cv2.VideoCapture(0)
HandGes = HandGesRec(mode="video")
while cap.isOpened():
    ret, frame = cap.read()
    HandGes.detect(frame)
    if HandGes.num_detected_hands>0:
        print(HandGes.get_gesture(0), HandGes.get_score_gesture(0))
    annotated_frame = HandGes.visualize(frame)
    cv2.imshow('annotated frame', annotated_frame)
    key = cv2.waitKey(1)&0xFF
    if key == ord('q'):
        break
cv2.destroyAllWindows()
HandGes.release()
cap.release()

# myhand_ges_simple_image.py
import cv2
from MediapipeHandGestureRecognition import MediapipeHandGestureRecognition as HandGesRec

img = cv2.imread("./img/standard/Balloon.bmp")
HandGes = HandGesRec(mode="image")
HandGes.detect(img)
if HandGes.num_detected_hands>0:
    print(HandGes.get_gesture(0), HandGes.get_score_gesture(0))
annotated_img = HandGes.visualize(img)

cv2.imshow("annotated_img", annotated_img)
cv2.waitKey(0)
HandGes.release()

⭕ PoseLandmark

• draw pose landmarks and segmentation mask on image
  

# mypose_simple.py
import os
os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0"
import cv2
from MediapipePoseLandmark import MediapipePoseLandmark as PoseLmk

cap = cv2.VideoCapture(0)
Pose = PoseLmk(mode="video")
while cap.isOpened():
    ret, frame = cap.read()
    Pose.detect(frame)
    masks = Pose.get_all_segmentation_masks()
    masked_frame = Pose.visualize_mask(frame, masks)
    annotated_frame = Pose.visualize(masked_frame)
    cv2.imshow('frame', annotated_frame)
    key = cv2.waitKey(1)&0xFF
    if key == ord('q'):
        break
cv2.destroyAllWindows()
Pose.release()
cap.release()

# mypose_simple_image.py
import cv2
from MediapipePoseLandmark import MediapipePoseLandmark as PoseLmk

img = cv2.imread("./img/standard/Balloon.bmp")
Pose = PoseLmk(mode="image")
Pose.detect(img)
annotated_img = Pose.visualize(img)
cv2.imshow("annotated_img", annotated_img)
cv2.waitKey(0)
cv2.destroyAllWindows()
Pose.release()

⭕ FaceLandmark

• draw face landmarks
  

# myface_simple.py
import os
os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0"
import cv2
from MediapipeFaceLandmark import MediapipeFaceLandmark as FaceLmk

cap = cv2.VideoCapture(0)
Face = FaceLmk(mode="video")
while cap.isOpened():
    ret, frame = cap.read()
    flipped_frame = cv2.flip(frame, 1)
    Face.detect(flipped_frame)
    annotated_frame = Face.visualize(flipped_frame)
    cv2.imshow('frame', annotated_frame)
    key = cv2.waitKey(1)&0xFF
    if key == ord('q'):
        break
cv2.destroyAllWindows()
Face.release()
cap.release()

# myface_simple_image.py
import cv2
from MediapipeFaceLandmark import MediapipeFaceLandmark as FaceLmk

img = cv2.imread("./img/standard/Balloon.bmp")
Face = FaceLmk(mode="image")
Face.detect(img)
annotated_img = Face.visualize(img)
cv2.imshow("annotated_img", annotated_img)
cv2.waitKey(0)
cv2.destroyAllWindows()
Face.release()

⭕ FaceDetection

• draw face bounding box, face keypoints, and detection score
  

# myface_dtc_simple.py
import os
os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0"
import cv2
from MediapipeFaceDetection import MediapipeFaceDetection as FaceDect

cap = cv2.VideoCapture(0)
Face = FaceDect(mode="video")
while cap.isOpened():
    ret, frame = cap.read()
    flipped_frame = cv2.flip(frame, 1)
    Face.detect(flipped_frame)
    annotated_frame = Face.visualize(flipped_frame)
    cv2.imshow('frame', annotated_frame)
    key = cv2.waitKey(1)&0xFF
    if key == ord('q'):
        break
cv2.destroyAllWindows()
Face.release()
cap.release()

# myface_dtc_simple_image.py
import cv2
from MediapipeFaceDetection import MediapipeFaceDetection as FaceDect

img = cv2.imread("./img/standard/Balloon.bmp")
Face = FaceDect(mode="image")
Face.detect(img)
annotated_img = Face.visualize(img)
cv2.imshow("annotated_img", annotated_img)
cv2.waitKey(0)
cv2.destroyAllWindows()
Face.release()

⭕ ObjectDetection

• draw the all object's bounding box, object name and detection score, and print the number of detected objects
  

# myobj_simple.py
import os
os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0"
import cv2
from MediapipeObjectDetection import MediapipeObjectDetection as ObjDetection

cap = cv2.VideoCapture(0)
Obj = ObjDetection(mode="video", score_threshold=0.5)
while cap.isOpened():
    ret, frame = cap.read()
    Obj.detect(frame)
    print(Obj.num_detected_objects)
    annotated_frame = Obj.visualize(frame)
    cv2.imshow('annotated frame', annotated_frame)
    key = cv2.waitKey(1)&0xFF
    if key == ord('q'):
        break
cv2.destroyAllWindows()
Obj.release()
cap.release()

# myobj_simple_image.py
import cv2
from MediapipeObjectDetection import MediapipeObjectDetection as ObjDetection

img = cv2.imread("./img/standard/Balloon.bmp")
Obj = ObjDetection(mode="image", score_threshold=0.5)
Obj.detect(img)
print(Obj.num_detected_objects)
annotated_frame = Obj.visualize(img)
cv2.imshow('annotated frame', annotated_frame)
cv2.waitKey(0)
cv2.destroyAllWindows()
Obj.release()

⭕ ImageSegmentation

• draw normalized segmentation masks, face skin mask, input image
  • dark blue:background, blue: hair, light blue: body_skin, yellow: face_skin, orange: clothes, red: others
    
  

# myseg_simple.py
import os
os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0"
import cv2
from MediapipeImageSegmentation import MediapipeImageSegmentation as ImgSeg

cap = cv2.VideoCapture(0)
Seg = ImgSeg(mode="video")
while cap.isOpened():
    ret, frame = cap.read()
    Seg.detect(frame)
    normalized_masks = Seg.get_normalized_masks()
    cv2.imshow('multiclass mask', cv2.applyColorMap(normalized_masks, cv2.COLORMAP_JET))
    face_skin_masks = Seg.get_segmentation_mask(Seg.FACE_SKIN)
    cv2.imshow('face skin', face_skin_masks)
    cv2.imshow('frame', frame)
    key = cv2.waitKey(1)&0xFF
    if key == ord('q'):
        break
cv2.destroyAllWindows()
Seg.release()
cap.release()

# myseg_simple_image.py
import cv2
from MediapipeImageSegmentation import MediapipeImageSegmentation as ImgSeg

img = cv2.imread("./img/standard/Balloon.bmp")
Seg = ImgSeg(mode="image")
Seg.detect(img)
normalized_masks = Seg.get_normalized_masks()
cv2.imshow('multiclass mask', cv2.applyColorMap(normalized_masks, cv2.COLORMAP_JET))
face_skin_masks = Seg.get_segmentation_mask(Seg.FACE_SKIN)
cv2.imshow('face skin', face_skin_masks)
cv2.imshow('frame', img)
cv2.waitKey(0)
cv2.destroyAllWindows()
Seg.release()