ตัวอย่างการทำระบบ Hand Tracking ของมือ แบบ Real-time ด้วย MediaPipe Library ภาษา Python ของ Google สำหรับผู้ที่สนใจทำระบบการตรวจจับท่าทางของมือ
ในบทเรียนนี้อาจจะต้องศึกษาบทเรียนก่อนหน้า:
- OpenCV Archives | Game & Mobile Development AR VR XR (daydev.com)
- การประมวลผลรู้จำใบหน้า Face Recognition ด้วย Python
- Python การใช้ Deep Learning ปรับภาพขาวดำเป็นภาพสี BW to Color ร่วมกับ OpenCV
- Machine Learning ใช้ Python และ OpenCV ทำระบบ Measuring Size ขนาดของวัตถุในภาพ
เพียงแค่รอบนี้เราจะเตียม Python ของเราให้พร้อมครับ สร้างไฟล์ว่า Hand Tracking หลังจากนั้นให้เราไปใช้ตัวอย่าง Library บน GitHub ที่ชื่อว่า MediaPipe ที่นี่ครับ:
Home – mediapipe (google.github.io)
MediaPipe เป็น Machine Learning หรือ Deep Learning ตัวหนึ่งจาก Google ใช้ในการจับท่าทางของมือหรือ Hand Tracking ข้อดีคือความรวดเร็วแบบ Real-time และความง่ายในการเรียกใช้งานได้ อีกทั้ง MediaPipe นั้นก็ค่อนข้างแม่นยำอยู่ระดับหนึ่งเลยในการจับ มือของเรา
เพิ่มเติมอีกเล็กน้อย MediaPipe ยังมีอีกหลาย Solutions เทียบเท่า Face Recognition, Gesture Pose, Knife สำหรับ AR
ทำการติดตั้งโดยพิมพ์คำสั่ง:
pip install mediapipe
ทำการติดตั้ง MediaPipe เพื่อใช้งานร่วมกับ OpenCV
ทำการประกาศ OpenCV และจับ Capture Video จาก WebCam จาก Source ตัวอย่างได้เลย
import copy
import argparse
import cv2 as cv
import numpy as np
import mediapipe as mp
def get_args():
parser = argparse.ArgumentParser()
parser.add_argument("--device", type=int, default=0)
parser.add_argument("--width", help='cap width', type=int, default=960)
parser.add_argument("--height", help='cap height', type=int, default=540)
parser.add_argument("--max_num_hands", type=int, default=2)
parser.add_argument("--min_detection_confidence",
help='min_detection_confidence',
type=float,
default=0.7)
parser.add_argument("--min_tracking_confidence",
help='min_tracking_confidence',
type=int,
default=0.5)
parser.add_argument('--use_brect', action='store_true')
args = parser.parse_args()
return args
def main():
# Parameter analysis #################################################################
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
max_num_hands = args.max_num_hands
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
use_brect = args.use_brect
# Camera preparation ###############################################################
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
# Load Model #############################################################
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(
max_num_hands=max_num_hands,
min_detection_confidence=min_detection_confidence,
min_tracking_confidence=min_tracking_confidence,
)
while True:
# Camera Capture #####################################################
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1) # ミラー表示
debug_image = copy.deepcopy(image)
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
results = hands.process(image)
if results.multi_hand_landmarks is not None:
for hand_landmarks, handedness in zip(results.multi_hand_landmarks,
results.multi_handedness):
cx, cy = calc_palm_moment(debug_image, hand_landmarks)
brect = calc_bounding_rect(debug_image, hand_landmarks)
debug_image = draw_landmarks(debug_image, cx, cy,
hand_landmarks, handedness)
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
cv.putText(debug_image, " ", (10, 30),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
key = cv.waitKey(1)
if key == 27: # ESC
break
cv.imshow('MediaPipe Hand Tracking', debug_image)
cap.release()
cv.destroyAllWindows()
def calc_palm_moment(image, landmarks):
image_width, image_height = image.shape[1], image.shape[0]
palm_array = np.empty((0, 2), int)
for index, landmark in enumerate(landmarks.landmark):
landmark_x = min(int(landmark.x * image_width), image_width - 1)
landmark_y = min(int(landmark.y * image_height), image_height - 1)
landmark_point = [np.array((landmark_x, landmark_y))]
if index == 0:
palm_array = np.append(palm_array, landmark_point, axis=0)
if index == 1:
palm_array = np.append(palm_array, landmark_point, axis=0)
if index == 5:
palm_array = np.append(palm_array, landmark_point, axis=0)
if index == 9:
palm_array = np.append(palm_array, landmark_point, axis=0)
if index == 13:
palm_array = np.append(palm_array, landmark_point, axis=0)
if index == 17:
palm_array = np.append(palm_array, landmark_point, axis=0)
M = cv.moments(palm_array)
cx, cy = 0, 0
if M['m00'] != 0:
cx = int(M['m10'] / M['m00'])
cy = int(M['m01'] / M['m00'])
return cx, cy
def calc_bounding_rect(image, landmarks):
image_width, image_height = image.shape[1], image.shape[0]
landmark_array = np.empty((0, 2), int)
for _, landmark in enumerate(landmarks.landmark):
landmark_x = min(int(landmark.x * image_width), image_width - 1)
landmark_y = min(int(landmark.y * image_height), image_height - 1)
landmark_point = [np.array((landmark_x, landmark_y))]
landmark_array = np.append(landmark_array, landmark_point, axis=0)
x, y, w, h = cv.boundingRect(landmark_array)
return [x, y, x + w, y + h]
def draw_landmarks(image, cx, cy, landmarks, handedness):
image_width, image_height = image.shape[1], image.shape[0]
landmark_point = []
for index, landmark in enumerate(landmarks.landmark):
if landmark.visibility < 0 or landmark.presence < 0:
continue
landmark_x = min(int(landmark.x * image_width), image_width - 1)
landmark_y = min(int(landmark.y * image_height), image_height - 1)
# landmark_z = landmark.z
landmark_point.append((landmark_x, landmark_y))
if index == 0:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 1:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 2:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 3:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 4:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
cv.circle(image, (landmark_x, landmark_y), 12, (0, 255, 0), 2)
if index == 5:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 6:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 7:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 8:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
cv.circle(image, (landmark_x, landmark_y), 12, (0, 255, 0), 2)
if index == 9:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 10:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 11:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 12:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
cv.circle(image, (landmark_x, landmark_y), 12, (0, 255, 0), 2)
if index == 13:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 14:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 15:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 16:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
cv.circle(image, (landmark_x, landmark_y), 12, (0, 255, 0), 2)
if index == 17:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 18:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 19:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 20:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
cv.circle(image, (landmark_x, landmark_y), 12, (0, 255, 0), 2)
if len(landmark_point) > 0:
cv.line(image, landmark_point[2], landmark_point[3], (0, 255, 0), 2)
cv.line(image, landmark_point[3], landmark_point[4], (0, 255, 0), 2)
cv.line(image, landmark_point[5], landmark_point[6], (0, 255, 0), 2)
cv.line(image, landmark_point[6], landmark_point[7], (0, 255, 0), 2)
cv.line(image, landmark_point[7], landmark_point[8], (0, 255, 0), 2)
cv.line(image, landmark_point[9], landmark_point[10], (0, 255, 0), 2)
cv.line(image, landmark_point[10], landmark_point[11], (0, 255, 0), 2)
cv.line(image, landmark_point[11], landmark_point[12], (0, 255, 0), 2)
cv.line(image, landmark_point[13], landmark_point[14], (0, 255, 0), 2)
cv.line(image, landmark_point[14], landmark_point[15], (0, 255, 0), 2)
cv.line(image, landmark_point[15], landmark_point[16], (0, 255, 0), 2)
cv.line(image, landmark_point[17], landmark_point[18], (0, 255, 0), 2)
cv.line(image, landmark_point[18], landmark_point[19], (0, 255, 0), 2)
cv.line(image, landmark_point[19], landmark_point[20], (0, 255, 0), 2)
cv.line(image, landmark_point[0], landmark_point[1], (0, 255, 0), 2)
cv.line(image, landmark_point[1], landmark_point[2], (0, 255, 0), 2)
cv.line(image, landmark_point[2], landmark_point[5], (0, 255, 0), 2)
cv.line(image, landmark_point[5], landmark_point[9], (0, 255, 0), 2)
cv.line(image, landmark_point[9], landmark_point[13], (0, 255, 0), 2)
cv.line(image, landmark_point[13], landmark_point[17], (0, 255, 0), 2)
cv.line(image, landmark_point[17], landmark_point[0], (0, 255, 0), 2)
if len(landmark_point) > 0:
cv.circle(image, (cx, cy), 12, (0, 255, 0), 2)
cv.putText(image, handedness.classification[0].label[0],
(cx - 6, cy + 6), cv.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0),
2, cv.LINE_AA)
return image
def draw_bounding_rect(use_brect, image, brect):
if use_brect:
cv.rectangle(image, (brect[0], brect[1]), (brect[2], brect[3]),
(0, 255, 0), 2)
return image
if __name__ == '__main__':
main()
Code ดูยาวอธิบายง่ายๆ
จะใช้การจับภาพจาก webcam แบบ Real-Time ต่อเนื่องโดยใช้ while loop แสดงผลต่อกันเป็นวิดีโอ
while True:
# Camera Capture #####################################################
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1) # ミラー表示
debug_image = copy.deepcopy(image)
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
results = hands.process(image)
if results.multi_hand_landmarks is not None:
for hand_landmarks, handedness in zip(results.multi_hand_landmarks,
results.multi_handedness):
cx, cy = calc_palm_moment(debug_image, hand_landmarks)
brect = calc_bounding_rect(debug_image, hand_landmarks)
debug_image = draw_landmarks(debug_image, cx, cy,
hand_landmarks, handedness)
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
cv.putText(debug_image, " ", (10, 30),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
key = cv.waitKey(1)
if key == 27: # ESC
break
cv.imshow('MediaPipe Hand Tracking', debug_image)
cap.release()
cv.destroyAllWindows()
จับ Palm connector มือของเราทันที
def draw_landmarks(image, cx, cy, landmarks, handedness):
image_width, image_height = image.shape[1], image.shape[0]
landmark_point = []
for index, landmark in enumerate(landmarks.landmark):
if landmark.visibility < 0 or landmark.presence < 0:
continue
landmark_x = min(int(landmark.x * image_width), image_width - 1)
landmark_y = min(int(landmark.y * image_height), image_height - 1)
# landmark_z = landmark.z
landmark_point.append((landmark_x, landmark_y))
if index == 0:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 1:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 2:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 3:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 4:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
cv.circle(image, (landmark_x, landmark_y), 12, (0, 255, 0), 2)
if index == 5:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 6:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 7:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 8:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
cv.circle(image, (landmark_x, landmark_y), 12, (0, 255, 0), 2)
if index == 9:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 10:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 11:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 12:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
cv.circle(image, (landmark_x, landmark_y), 12, (0, 255, 0), 2)
if index == 13:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 14:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 15:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 16:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
cv.circle(image, (landmark_x, landmark_y), 12, (0, 255, 0), 2)
if index == 17:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 18:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 19:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
if index == 20:
cv.circle(image, (landmark_x, landmark_y), 5, (0, 255, 0), 2)
cv.circle(image, (landmark_x, landmark_y), 12, (0, 255, 0), 2)
if len(landmark_point) > 0:
cv.line(image, landmark_point[2], landmark_point[3], (0, 255, 0), 2)
cv.line(image, landmark_point[3], landmark_point[4], (0, 255, 0), 2)
cv.line(image, landmark_point[5], landmark_point[6], (0, 255, 0), 2)
cv.line(image, landmark_point[6], landmark_point[7], (0, 255, 0), 2)
cv.line(image, landmark_point[7], landmark_point[8], (0, 255, 0), 2)
cv.line(image, landmark_point[9], landmark_point[10], (0, 255, 0), 2)
cv.line(image, landmark_point[10], landmark_point[11], (0, 255, 0), 2)
cv.line(image, landmark_point[11], landmark_point[12], (0, 255, 0), 2)
cv.line(image, landmark_point[13], landmark_point[14], (0, 255, 0), 2)
cv.line(image, landmark_point[14], landmark_point[15], (0, 255, 0), 2)
cv.line(image, landmark_point[15], landmark_point[16], (0, 255, 0), 2)
cv.line(image, landmark_point[17], landmark_point[18], (0, 255, 0), 2)
cv.line(image, landmark_point[18], landmark_point[19], (0, 255, 0), 2)
cv.line(image, landmark_point[19], landmark_point[20], (0, 255, 0), 2)
cv.line(image, landmark_point[0], landmark_point[1], (0, 255, 0), 2)
cv.line(image, landmark_point[1], landmark_point[2], (0, 255, 0), 2)
cv.line(image, landmark_point[2], landmark_point[5], (0, 255, 0), 2)
cv.line(image, landmark_point[5], landmark_point[9], (0, 255, 0), 2)
cv.line(image, landmark_point[9], landmark_point[13], (0, 255, 0), 2)
cv.line(image, landmark_point[13], landmark_point[17], (0, 255, 0), 2)
cv.line(image, landmark_point[17], landmark_point[0], (0, 255, 0), 2)
if len(landmark_point) > 0:
cv.circle(image, (cx, cy), 12, (0, 255, 0), 2)
cv.putText(image, handedness.classification[0].label[0],
(cx - 6, cy + 6), cv.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0),
2, cv.LINE_AA)
return image
ค่อนข้างมีการประมวลผลเยอะ มันจะวิ่งหาตำแหน่ง Landmark บนมือของเราระบุรายละเอียด ขนาดที่แตกต่างกันของมือแต่ละข้าง เก็บข้อมูลที่ได้มาจาก LandMark ไว้ในตัวแปร หากสังเกตจะเห็นว่ามีหลาย landmark มาก
เอาละเราลองมา Track และแสดงผลท่าทางของมือกันหน่อยมี Video นะ
บทเรียนหน้าเป็น Holistic จับ Body และ ใบหน้า Source ก็ใน Github นั่นแหละครับ เลยรีวิวไม่เยอะมาก Copy ก็ใช้ได้แล้ว!
