File size: 6,132 Bytes
a58bc70 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 |
# This file is part of OpenCV Zoo project.
# It is subject to the license terms in the LICENSE file found in the same directory.
#
# Copyright (C) 2022, Shenzhen Institute of Artificial Intelligence and Robotics for Society, all rights reserved.
# Third party copyrights are property of their respective owners.
import numpy as np
import cv2 as cv
class FacialExpressionRecog:
def __init__(self, modelPath, backendId=0, targetId=0):
self._modelPath = modelPath
self._backendId = backendId
self._targetId = targetId
self._model = cv.dnn.readNet(self._modelPath)
self._model.setPreferableBackend(self._backendId)
self._model.setPreferableTarget(self._targetId)
self._align_model = FaceAlignment()
self._inputNames = 'data'
self._outputNames = ['label']
self._inputSize = [112, 112]
self._mean = np.array([0.5, 0.5, 0.5])[np.newaxis, np.newaxis, :]
self._std = np.array([0.5, 0.5, 0.5])[np.newaxis, np.newaxis, :]
@property
def name(self):
return self.__class__.__name__
def setBackendAndTarget(self, backendId, targetId):
self._backendId = backendId
self._targetId = targetId
self._model.setPreferableBackend(self._backendId)
self._model.setPreferableTarget(self._targetId)
def _preprocess(self, image, bbox):
if bbox is not None:
image = self._align_model.get_align_image(image, bbox[4:].reshape(-1, 2))
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
image = image.astype(np.float32, copy=False) / 255.0
image -= self._mean
image /= self._std
return cv.dnn.blobFromImage(image)
def infer(self, image, bbox=None):
# Preprocess
inputBlob = self._preprocess(image, bbox)
# Forward
self._model.setInput(inputBlob, self._inputNames)
outputBlob = self._model.forward(self._outputNames)
# Postprocess
results = self._postprocess(outputBlob)
return results
def _postprocess(self, outputBlob):
result = np.argmax(outputBlob[0], axis=1).astype(np.uint8)
return result
@staticmethod
def getDesc(ind):
_expression_enum = ["angry", "disgust", "fearful", "happy", "neutral", "sad", "surprised"]
return _expression_enum[ind]
class FaceAlignment():
def __init__(self, reflective=False):
self._std_points = np.array([[38.2946, 51.6963], [73.5318, 51.5014], [56.0252, 71.7366], [41.5493, 92.3655], [70.7299, 92.2041]])
self.reflective = reflective
def __tformfwd(self, trans, uv):
uv = np.hstack((uv, np.ones((uv.shape[0], 1))))
xy = np.dot(uv, trans)
xy = xy[:, 0:-1]
return xy
def __tforminv(self, trans, uv):
Tinv = np.linalg.inv(trans)
xy = self.__tformfwd(Tinv, uv)
return xy
def __findNonreflectiveSimilarity(self, uv, xy, options=None):
options = {"K": 2}
K = options["K"]
M = xy.shape[0]
x = xy[:, 0].reshape((-1, 1)) # use reshape to keep a column vector
y = xy[:, 1].reshape((-1, 1)) # use reshape to keep a column vector
# print '--->x, y:\n', x, y
tmp1 = np.hstack((x, y, np.ones((M, 1)), np.zeros((M, 1))))
tmp2 = np.hstack((y, -x, np.zeros((M, 1)), np.ones((M, 1))))
X = np.vstack((tmp1, tmp2))
# print '--->X.shape: ', X.shape
# print 'X:\n', X
u = uv[:, 0].reshape((-1, 1)) # use reshape to keep a column vector
v = uv[:, 1].reshape((-1, 1)) # use reshape to keep a column vector
U = np.vstack((u, v))
# print '--->U.shape: ', U.shape
# print 'U:\n', U
# We know that X * r = U
if np.linalg.matrix_rank(X) >= 2 * K:
r, _, _, _ = np.linalg.lstsq(X, U, rcond=-1)
# print(r, X, U, sep="\n")
r = np.squeeze(r)
else:
raise Exception("cp2tform:twoUniquePointsReq")
sc = r[0]
ss = r[1]
tx = r[2]
ty = r[3]
Tinv = np.array([[sc, -ss, 0], [ss, sc, 0], [tx, ty, 1]])
T = np.linalg.inv(Tinv)
T[:, 2] = np.array([0, 0, 1])
return T, Tinv
def __findSimilarity(self, uv, xy, options=None):
options = {"K": 2}
# uv = np.array(uv)
# xy = np.array(xy)
# Solve for trans1
trans1, trans1_inv = self.__findNonreflectiveSimilarity(uv, xy, options)
# manually reflect the xy data across the Y-axis
xyR = xy
xyR[:, 0] = -1 * xyR[:, 0]
# Solve for trans2
trans2r, trans2r_inv = self.__findNonreflectiveSimilarity(uv, xyR, options)
# manually reflect the tform to undo the reflection done on xyR
TreflectY = np.array([[-1, 0, 0], [0, 1, 0], [0, 0, 1]])
trans2 = np.dot(trans2r, TreflectY)
# Figure out if trans1 or trans2 is better
xy1 = self.__tformfwd(trans1, uv)
norm1 = np.linalg.norm(xy1 - xy)
xy2 = self.__tformfwd(trans2, uv)
norm2 = np.linalg.norm(xy2 - xy)
if norm1 <= norm2:
return trans1, trans1_inv
else:
trans2_inv = np.linalg.inv(trans2)
return trans2, trans2_inv
def __get_similarity_transform(self, src_pts, dst_pts):
if self.reflective:
trans, trans_inv = self.__findSimilarity(src_pts, dst_pts)
else:
trans, trans_inv = self.__findNonreflectiveSimilarity(src_pts, dst_pts)
return trans, trans_inv
def __cvt_tform_mat_for_cv2(self, trans):
cv2_trans = trans[:, 0:2].T
return cv2_trans
def get_similarity_transform_for_cv2(self, src_pts, dst_pts):
trans, trans_inv = self.__get_similarity_transform(src_pts, dst_pts)
cv2_trans = self.__cvt_tform_mat_for_cv2(trans)
return cv2_trans, trans
def get_align_image(self, image, lm5_points):
assert lm5_points is not None
tfm, trans = self.get_similarity_transform_for_cv2(lm5_points, self._std_points)
return cv.warpAffine(image, tfm, (112, 112))
|