# Copyright (c) OpenMMLab. All rights reserved. from itertools import product from typing import Optional, Tuple, Union import numpy as np from scipy.spatial.distance import cdist def generate_3d_gaussian_heatmaps( heatmap_size: Tuple[int, int, int], keypoints: np.ndarray, keypoints_visible: np.ndarray, sigma: Union[float, Tuple[float], np.ndarray], image_size: Tuple[int, int], heatmap3d_depth_bound: float = 400.0, joint_indices: Optional[list] = None, max_bound: float = 1.0, use_different_joint_weights: bool = False, dataset_keypoint_weights: Optional[np.ndarray] = None ) -> Tuple[np.ndarray, np.ndarray]: """Generate 3d gaussian heatmaps of keypoints. Args: heatmap_size (Tuple[int, int]): Heatmap size in [W, H, D] keypoints (np.ndarray): Keypoint coordinates in shape (N, K, C) keypoints_visible (np.ndarray): Keypoint visibilities in shape (N, K) sigma (float or List[float]): A list of sigma values of the Gaussian heatmap for each instance. If sigma is given as a single float value, it will be expanded into a tuple image_size (Tuple[int, int]): Size of input image. heatmap3d_depth_bound (float): Boundary for 3d heatmap depth. Default: 400.0. joint_indices (List[int], optional): Indices of joints used for heatmap generation. If None (default) is given, all joints will be used. Default: ``None``. max_bound (float): The maximal value of heatmap. Default: 1.0. use_different_joint_weights (bool): Whether to use different joint weights. Default: ``False``. dataset_keypoint_weights (np.ndarray, optional): Keypoints weight in shape (K, ). Returns: tuple: - heatmaps (np.ndarray): The generated heatmap in shape (K * D, H, W) where [W, H, D] is the `heatmap_size` - keypoint_weights (np.ndarray): The target weights in shape (N, K) """ W, H, D = heatmap_size # select the joints used for target generation if joint_indices is not None: keypoints = keypoints[:, joint_indices, ...] keypoints_visible = keypoints_visible[:, joint_indices, ...] N, K, _ = keypoints.shape heatmaps = np.zeros([K, D, H, W], dtype=np.float32) keypoint_weights = keypoints_visible.copy() if isinstance(sigma, (int, float)): sigma = (sigma, ) * N for n in range(N): # 3-sigma rule radius = sigma[n] * 3 # joint location in heatmap coordinates mu_x = keypoints[n, :, 0] * W / image_size[0] # (K, ) mu_y = keypoints[n, :, 1] * H / image_size[1] mu_z = (keypoints[n, :, 2] / heatmap3d_depth_bound + 0.5) * D keypoint_weights[n, ...] = keypoint_weights[n, ...] * (mu_z >= 0) * ( mu_z < D) if use_different_joint_weights: keypoint_weights[ n] = keypoint_weights[n] * dataset_keypoint_weights # xy grid gaussian_size = 2 * radius + 1 # get neighboring voxels coordinates x = y = z = np.arange(gaussian_size, dtype=np.float32) - radius zz, yy, xx = np.meshgrid(z, y, x) xx = np.expand_dims(xx, axis=0) yy = np.expand_dims(yy, axis=0) zz = np.expand_dims(zz, axis=0) mu_x = np.expand_dims(mu_x, axis=(-1, -2, -3)) mu_y = np.expand_dims(mu_y, axis=(-1, -2, -3)) mu_z = np.expand_dims(mu_z, axis=(-1, -2, -3)) xx, yy, zz = xx + mu_x, yy + mu_y, zz + mu_z local_size = xx.shape[1] # round the coordinates xx = xx.round().clip(0, W - 1) yy = yy.round().clip(0, H - 1) zz = zz.round().clip(0, D - 1) # compute the target value near joints gaussian = np.exp(-((xx - mu_x)**2 + (yy - mu_y)**2 + (zz - mu_z)**2) / (2 * sigma[n]**2)) # put the local target value to the full target heatmap idx_joints = np.tile( np.expand_dims(np.arange(K), axis=(-1, -2, -3)), [1, local_size, local_size, local_size]) idx = np.stack([idx_joints, zz, yy, xx], axis=-1).astype(int).reshape(-1, 4) heatmaps[idx[:, 0], idx[:, 1], idx[:, 2], idx[:, 3]] = np.maximum( heatmaps[idx[:, 0], idx[:, 1], idx[:, 2], idx[:, 3]], gaussian.reshape(-1)) heatmaps = (heatmaps * max_bound).reshape(-1, H, W) return heatmaps, keypoint_weights def generate_gaussian_heatmaps( heatmap_size: Tuple[int, int], keypoints: np.ndarray, keypoints_visible: np.ndarray, sigma: Union[float, Tuple[float], np.ndarray], ) -> Tuple[np.ndarray, np.ndarray]: """Generate gaussian heatmaps of keypoints. Args: heatmap_size (Tuple[int, int]): Heatmap size in [W, H] keypoints (np.ndarray): Keypoint coordinates in shape (N, K, D) keypoints_visible (np.ndarray): Keypoint visibilities in shape (N, K) sigma (float or List[float]): A list of sigma values of the Gaussian heatmap for each instance. If sigma is given as a single float value, it will be expanded into a tuple Returns: tuple: - heatmaps (np.ndarray): The generated heatmap in shape (K, H, W) where [W, H] is the `heatmap_size` - keypoint_weights (np.ndarray): The target weights in shape (N, K) """ N, K, _ = keypoints.shape W, H = heatmap_size heatmaps = np.zeros((K, H, W), dtype=np.float32) keypoint_weights = keypoints_visible.copy() if isinstance(sigma, (int, float)): sigma = (sigma, ) * N for n in range(N): # 3-sigma rule radius = sigma[n] * 3 # xy grid gaussian_size = 2 * radius + 1 x = np.arange(0, gaussian_size, 1, dtype=np.float32) y = x[:, None] x0 = y0 = gaussian_size // 2 for k in range(K): # skip unlabled keypoints if keypoints_visible[n, k] < 0.5: continue # get gaussian center coordinates mu = (keypoints[n, k] + 0.5).astype(np.int64) # check that the gaussian has in-bounds part left, top = (mu - radius).astype(np.int64) right, bottom = (mu + radius + 1).astype(np.int64) if left >= W or top >= H or right < 0 or bottom < 0: keypoint_weights[n, k] = 0 continue # The gaussian is not normalized, # we want the center value to equal 1 gaussian = np.exp(-((x - x0)**2 + (y - y0)**2) / (2 * sigma[n]**2)) # valid range in gaussian g_x1 = max(0, -left) g_x2 = min(W, right) - left g_y1 = max(0, -top) g_y2 = min(H, bottom) - top # valid range in heatmap h_x1 = max(0, left) h_x2 = min(W, right) h_y1 = max(0, top) h_y2 = min(H, bottom) heatmap_region = heatmaps[k, h_y1:h_y2, h_x1:h_x2] gaussian_regsion = gaussian[g_y1:g_y2, g_x1:g_x2] _ = np.maximum( heatmap_region, gaussian_regsion, out=heatmap_region) return heatmaps, keypoint_weights def generate_unbiased_gaussian_heatmaps( heatmap_size: Tuple[int, int], keypoints: np.ndarray, keypoints_visible: np.ndarray, sigma: float, ) -> Tuple[np.ndarray, np.ndarray]: """Generate gaussian heatmaps of keypoints using `Dark Pose`_. Args: heatmap_size (Tuple[int, int]): Heatmap size in [W, H] keypoints (np.ndarray): Keypoint coordinates in shape (N, K, D) keypoints_visible (np.ndarray): Keypoint visibilities in shape (N, K) Returns: tuple: - heatmaps (np.ndarray): The generated heatmap in shape (K, H, W) where [W, H] is the `heatmap_size` - keypoint_weights (np.ndarray): The target weights in shape (N, K) .. _`Dark Pose`: https://arxiv.org/abs/1910.06278 """ N, K, _ = keypoints.shape W, H = heatmap_size heatmaps = np.zeros((K, H, W), dtype=np.float32) keypoint_weights = keypoints_visible.copy() # 3-sigma rule radius = sigma * 3 # xy grid x = np.arange(0, W, 1, dtype=np.float32) y = np.arange(0, H, 1, dtype=np.float32)[:, None] for n, k in product(range(N), range(K)): # skip unlabled keypoints if keypoints_visible[n, k] < 0.5: continue mu = keypoints[n, k] # check that the gaussian has in-bounds part left, top = mu - radius right, bottom = mu + radius + 1 if left >= W or top >= H or right < 0 or bottom < 0: keypoint_weights[n, k] = 0 continue gaussian = np.exp(-((x - mu[0])**2 + (y - mu[1])**2) / (2 * sigma**2)) _ = np.maximum(gaussian, heatmaps[k], out=heatmaps[k]) return heatmaps, keypoint_weights def generate_udp_gaussian_heatmaps( heatmap_size: Tuple[int, int], keypoints: np.ndarray, keypoints_visible: np.ndarray, sigma, keypoints_visibility: np.ndarray, increase_sigma_with_padding: bool = False, ) -> Tuple[np.ndarray, np.ndarray]: """Generate gaussian heatmaps of keypoints using `UDP`_. Args: heatmap_size (Tuple[int, int]): Heatmap size in [W, H] keypoints (np.ndarray): Keypoint coordinates in shape (N, K, D) keypoints_visible (np.ndarray): Keypoint visibilities in shape (N, K) sigma (float): The sigma value of the Gaussian heatmap keypoints_visibility (np.ndarray): The visibility bit for each keypoint (N, K) increase_sigma_with_padding (bool): Whether to increase the sigma value with padding. Default: False Returns: tuple: - heatmaps (np.ndarray): The generated heatmap in shape (K, H, W) where [W, H] is the `heatmap_size` - keypoint_weights (np.ndarray): The target weights in shape (N, K) .. _`UDP`: https://arxiv.org/abs/1911.07524 """ N, K, _ = keypoints.shape W, H = heatmap_size heatmaps = np.zeros((K, H, W), dtype=np.float32) keypoint_weights = keypoints_visible.copy() if isinstance(sigma, (int, float)): scaled_sigmas = sigma * np.ones((N, K), dtype=np.float32) sigmas = np.array([sigma] * K).reshape(1, -1).repeat(N, axis=0) else: scaled_sigmas = np.array(sigma).reshape(1, -1).repeat(N, axis=0) sigmas = np.array(sigma).reshape(1, -1).repeat(N, axis=0) scales_arr = np.ones((N, K), dtype=np.float32) if increase_sigma_with_padding: diag = np.sqrt(W**2 + H**2) for n in range(N): image_kpts = keypoints[n, :].squeeze() vis_kpts = image_kpts[keypoints_visibility[n, :] > 0.5] # Compute the distance between img_kpts and visible_kpts if vis_kpts.size == 0: min_dists = np.ones(image_kpts.shape[0]) * diag else: dists = cdist(image_kpts, vis_kpts, metric='euclidean') min_dists = np.min(dists, axis=1) scales = min_dists / diag * 2.0 # Maximum distance (diagonal) results in .0*sigma scales_arr[n, :] = scales scaled_sigmas[n, :] = sigma * (1+scales) # print(scales_arr) # print(scaled_sigmas) for n, k in product(range(N), range(K)): scaled_sigma = scaled_sigmas[n, k] # skip unlabled keypoints if keypoints_visible[n, k] < 0.5: continue # 3-sigma rule radius = scaled_sigma * 3 # xy grid gaussian_size = 2 * radius + 1 x = np.arange(0, gaussian_size, 1, dtype=np.float32) y = x[:, None] mu = (keypoints[n, k] + 0.5).astype(np.int64) # check that the gaussian has in-bounds part left, top = (mu - radius).round().astype(np.int64) right, bottom = (mu + radius + 1).round().astype(np.int64) # left, top = (mu - radius).astype(np.int64) # right, bottom = (mu + radius + 1).astype(np.int64) if left >= W or top >= H or right < 0 or bottom < 0: keypoint_weights[n, k] = 0 continue mu_ac = keypoints[n, k] x0 = y0 = gaussian_size // 2 x0 += mu_ac[0] - mu[0] y0 += mu_ac[1] - mu[1] gaussian = np.exp(-((x - x0)**2 + (y - y0)**2) / (2 * scaled_sigma**2)) # Normalize Gaussian such that scaled_sigma = sigma is the norm gaussian = gaussian / (scaled_sigma / sigmas[n, k]) # valid range in gaussian g_x1 = max(0, -left) g_x2 = min(W, right) - left g_y1 = max(0, -top) g_y2 = min(H, bottom) - top # valid range in heatmap h_x1 = max(0, left) h_x2 = min(W, right) h_y1 = max(0, top) h_y2 = min(H, bottom) # breakpoint() heatmap_region = heatmaps[k, h_y1:h_y2, h_x1:h_x2] gaussian_regsion = gaussian[g_y1:g_y2, g_x1:g_x2] _ = np.maximum(heatmap_region, gaussian_regsion, out=heatmap_region) return heatmaps, keypoint_weights def generate_onehot_heatmaps( heatmap_size: Tuple[int, int], keypoints: np.ndarray, keypoints_visible: np.ndarray, sigma, keypoints_visibility: np.ndarray, increase_sigma_with_padding: bool = False, ) -> Tuple[np.ndarray, np.ndarray]: """Generate gaussian heatmaps of keypoints using `UDP`_. Args: heatmap_size (Tuple[int, int]): Heatmap size in [W, H] keypoints (np.ndarray): Keypoint coordinates in shape (N, K, D) keypoints_visible (np.ndarray): Keypoint visibilities in shape (N, K) sigma (float): The sigma value of the Gaussian heatmap keypoints_visibility (np.ndarray): The visibility bit for each keypoint (N, K) increase_sigma_with_padding (bool): Whether to increase the sigma value with padding. Default: False Returns: tuple: - heatmaps (np.ndarray): The generated heatmap in shape (K, H, W) where [W, H] is the `heatmap_size` - keypoint_weights (np.ndarray): The target weights in shape (N, K) .. _`UDP`: https://arxiv.org/abs/1911.07524 """ N, K, _ = keypoints.shape W, H = heatmap_size heatmaps = np.zeros((K, H, W), dtype=np.float32) keypoint_weights = keypoints_visible.copy() for n, k in product(range(N), range(K)): # skip unlabled keypoints if keypoints_visible[n, k] < 0.5: continue mu = (keypoints[n, k] + 0.5).astype(np.int64) if mu[0] < 0 or mu[0] >= W or mu[1] < 0 or mu[1] >= H: keypoint_weights[n, k] = 0 continue heatmaps[k, mu[1], mu[0]] = 1 return heatmaps, keypoint_weights