trellis/trainers/vae/structured_latent_vae_mesh_dec.py

from typing import *
import copy
import torch
from torch.utils.data import DataLoader
import numpy as np
from easydict import EasyDict as edict
import utils3d.torch

from ..basic import BasicTrainer
from ...representations import MeshExtractResult
from ...renderers import MeshRenderer
from ...modules.sparse import SparseTensor
from ...utils.loss_utils import l1_loss, smooth_l1_loss, ssim, lpips
from ...utils.data_utils import recursive_to_device


class SLatVaeMeshDecoderTrainer(BasicTrainer):
    """
    Trainer for structured latent VAE Mesh Decoder.
    
    Args:
        models (dict[str, nn.Module]): Models to train.
        dataset (torch.utils.data.Dataset): Dataset.
        output_dir (str): Output directory.
        load_dir (str): Load directory.
        step (int): Step to load.
        batch_size (int): Batch size.
        batch_size_per_gpu (int): Batch size per GPU. If specified, batch_size will be ignored.
        batch_split (int): Split batch with gradient accumulation.
        max_steps (int): Max steps.
        optimizer (dict): Optimizer config.
        lr_scheduler (dict): Learning rate scheduler config.
        elastic (dict): Elastic memory management config.
        grad_clip (float or dict): Gradient clip config.
        ema_rate (float or list): Exponential moving average rates.
        fp16_mode (str): FP16 mode.
            - None: No FP16.
            - 'inflat_all': Hold a inflated fp32 master param for all params.
            - 'amp': Automatic mixed precision.
        fp16_scale_growth (float): Scale growth for FP16 gradient backpropagation.
        finetune_ckpt (dict): Finetune checkpoint.
        log_param_stats (bool): Log parameter stats.
        i_print (int): Print interval.
        i_log (int): Log interval.
        i_sample (int): Sample interval.
        i_save (int): Save interval.
        i_ddpcheck (int): DDP check interval.
        
        loss_type (str): Loss type. Can be 'l1', 'l2'
        lambda_ssim (float): SSIM loss weight.
        lambda_lpips (float): LPIPS loss weight.
    """
    
    def __init__(
        self,
        *args,
        depth_loss_type: str = 'l1',
        lambda_depth: int = 1,
        lambda_ssim: float = 0.2,
        lambda_lpips: float = 0.2,
        lambda_tsdf: float = 0.01,
        lambda_color: float = 0.1,
        **kwargs
    ):
        super().__init__(*args, **kwargs)
        self.depth_loss_type = depth_loss_type
        self.lambda_depth = lambda_depth
        self.lambda_ssim = lambda_ssim
        self.lambda_lpips = lambda_lpips
        self.lambda_tsdf = lambda_tsdf
        self.lambda_color = lambda_color
        self.use_color = self.lambda_color > 0
        
        self._init_renderer()
        
    def _init_renderer(self):
        rendering_options = {"near" : 1,
                             "far" : 3}
        self.renderer = MeshRenderer(rendering_options, device=self.device)
        
    def _render_batch(self, reps: List[MeshExtractResult], extrinsics: torch.Tensor, intrinsics: torch.Tensor,
                      return_types=['mask', 'normal', 'depth']) -> Dict[str, torch.Tensor]:
        """
        Render a batch of representations.

        Args:
            reps: The dictionary of lists of representations.
            extrinsics: The [N x 4 x 4] tensor of extrinsics.
            intrinsics: The [N x 3 x 3] tensor of intrinsics.
            return_types: vary in ['mask', 'normal', 'depth', 'normal_map', 'color']
            
        Returns: 
            a dict with
                reg_loss : [N] tensor of regularization losses
                mask : [N x 1 x H x W] tensor of rendered masks
                normal : [N x 3 x H x W] tensor of rendered normals
                depth : [N x 1 x H x W] tensor of rendered depths
        """
        ret = {k : [] for k in return_types}
        for i, rep in enumerate(reps):
            out_dict = self.renderer.render(rep, extrinsics[i], intrinsics[i], return_types=return_types)
            for k in out_dict:
                ret[k].append(out_dict[k][None] if k in ['mask', 'depth'] else out_dict[k])
        for k in ret:
            ret[k] = torch.stack(ret[k])
        return ret
    
    @staticmethod
    def _tsdf_reg_loss(rep: MeshExtractResult, depth_map: torch.Tensor, extrinsics: torch.Tensor, intrinsics: torch.Tensor) -> torch.Tensor:
        # Calculate tsdf
        with torch.no_grad():
            # Project points to camera and calculate pseudo-sdf as difference between gt depth and projected depth
            projected_pts, pts_depth = utils3d.torch.project_cv(extrinsics=extrinsics, intrinsics=intrinsics, points=rep.tsdf_v)
            projected_pts = (projected_pts - 0.5) * 2.0
            depth_map_res = depth_map.shape[1]
            gt_depth = torch.nn.functional.grid_sample(depth_map.reshape(1, 1, depth_map_res, depth_map_res), 
            projected_pts.reshape(1, 1, -1, 2), mode='bilinear', padding_mode='border', align_corners=True)
            pseudo_sdf = gt_depth.flatten() - pts_depth.flatten()
            # Truncate pseudo-sdf
            delta = 1 / rep.res * 3.0
            trunc_mask = pseudo_sdf > -delta
        
        # Loss
        gt_tsdf = pseudo_sdf[trunc_mask]
        tsdf = rep.tsdf_s.flatten()[trunc_mask]
        gt_tsdf = torch.clamp(gt_tsdf, -delta, delta)
        return torch.mean((tsdf - gt_tsdf) ** 2)
    
    def _calc_tsdf_loss(self, reps : list[MeshExtractResult], depth_maps, extrinsics, intrinsics) -> torch.Tensor:
        tsdf_loss = 0.0
        for i, rep in enumerate(reps):
            tsdf_loss += self._tsdf_reg_loss(rep, depth_maps[i], extrinsics[i], intrinsics[i])
        return tsdf_loss / len(reps)
    
    @torch.no_grad()
    def _flip_normal(self, normal: torch.Tensor, extrinsics: torch.Tensor, intrinsics: torch.Tensor) -> torch.Tensor:
        """
        Flip normal to align with camera.
        """
        normal = normal * 2.0 - 1.0
        R = torch.zeros_like(extrinsics)
        R[:, :3, :3] = extrinsics[:, :3, :3]
        R[:, 3, 3] = 1.0
        view_dir = utils3d.torch.unproject_cv(
            utils3d.torch.image_uv(*normal.shape[-2:], device=self.device).reshape(1, -1, 2),
            torch.ones(*normal.shape[-2:], device=self.device).reshape(1, -1),
            R, intrinsics
        ).reshape(-1, *normal.shape[-2:], 3).permute(0, 3, 1, 2)
        unflip = (normal * view_dir).sum(1, keepdim=True) < 0
        normal *= unflip * 2.0 - 1.0
        return (normal + 1.0) / 2.0
    
    def _perceptual_loss(self, gt: torch.Tensor, pred: torch.Tensor, name: str) -> Dict[str, torch.Tensor]:
        """
        Combination of L1, SSIM, and LPIPS loss.
        """
        if gt.shape[1] != 3:
            assert gt.shape[-1] == 3
            gt = gt.permute(0, 3, 1, 2)
        if pred.shape[1] != 3:
            assert pred.shape[-1] == 3
            pred = pred.permute(0, 3, 1, 2)
        terms = {
            f"{name}_loss" : l1_loss(gt, pred),
            f"{name}_loss_ssim" : 1 - ssim(gt, pred),
            f"{name}_loss_lpips" : lpips(gt, pred)
        }
        terms[f"{name}_loss_perceptual"] = terms[f"{name}_loss"] + terms[f"{name}_loss_ssim"] * self.lambda_ssim + terms[f"{name}_loss_lpips"] * self.lambda_lpips
        return terms
    
    def geometry_losses(
        self,
        reps: List[MeshExtractResult],
        mesh: List[Dict],
        normal_map: torch.Tensor,
        extrinsics: torch.Tensor,
        intrinsics: torch.Tensor,
    ):
        with torch.no_grad():
            gt_meshes = []
            for i in range(len(reps)):
                gt_mesh = MeshExtractResult(mesh[i]['vertices'].to(self.device), mesh[i]['faces'].to(self.device))
                gt_meshes.append(gt_mesh)
            target = self._render_batch(gt_meshes, extrinsics, intrinsics, return_types=['mask', 'depth', 'normal'])
            target['normal'] = self._flip_normal(target['normal'], extrinsics, intrinsics)
                
        terms = edict(geo_loss = 0.0)
        if self.lambda_tsdf > 0:
            tsdf_loss = self._calc_tsdf_loss(reps, target['depth'], extrinsics, intrinsics)
            terms['tsdf_loss'] = tsdf_loss
            terms['geo_loss'] += tsdf_loss * self.lambda_tsdf
        
        return_types = ['mask', 'depth', 'normal', 'normal_map'] if self.use_color else ['mask', 'depth', 'normal']
        buffer = self._render_batch(reps, extrinsics, intrinsics, return_types=return_types)
        
        success_mask = torch.tensor([rep.success for rep in reps], device=self.device)
        if success_mask.sum() != 0:
            for k, v in buffer.items():
                buffer[k] = v[success_mask]
            for k, v in target.items():
                target[k] = v[success_mask]
            
            terms['mask_loss'] = l1_loss(buffer['mask'], target['mask']) 
            if self.depth_loss_type == 'l1':
                terms['depth_loss'] = l1_loss(buffer['depth'] * target['mask'], target['depth'] * target['mask'])
            elif self.depth_loss_type == 'smooth_l1':
                terms['depth_loss'] = smooth_l1_loss(buffer['depth'] * target['mask'], target['depth'] * target['mask'], beta=1.0 / (2 * reps[0].res))
            else:
                raise ValueError(f"Unsupported depth loss type: {self.depth_loss_type}")
            terms.update(self._perceptual_loss(buffer['normal'] * target['mask'], target['normal'] * target['mask'], 'normal'))
            terms['geo_loss'] = terms['geo_loss'] + terms['mask_loss'] + terms['depth_loss'] * self.lambda_depth + terms['normal_loss_perceptual']
            if self.use_color and normal_map is not None:
                terms.update(self._perceptual_loss(normal_map[success_mask], buffer['normal_map'], 'normal_map'))
                terms['geo_loss'] = terms['geo_loss'] + terms['normal_map_loss_perceptual'] * self.lambda_color
                
        return terms
      
    def color_losses(self, reps, image, alpha, extrinsics, intrinsics):
        terms = edict(color_loss = torch.tensor(0.0, device=self.device))
        buffer = self._render_batch(reps, extrinsics, intrinsics, return_types=['color'])
        success_mask = torch.tensor([rep.success for rep in reps], device=self.device)
        if success_mask.sum() != 0:
            terms.update(self._perceptual_loss((image * alpha[:, None])[success_mask], buffer['color'][success_mask], 'color'))
            terms['color_loss'] = terms['color_loss'] + terms['color_loss_perceptual'] * self.lambda_color
        return terms
    
    def training_losses(
        self,
        latents: SparseTensor,
        image: torch.Tensor,
        alpha: torch.Tensor,
        mesh: List[Dict],
        extrinsics: torch.Tensor,
        intrinsics: torch.Tensor,
        normal_map: torch.Tensor = None,
    ) -> Tuple[Dict, Dict]:
        """
        Compute training losses.

        Args:
            latents: The [N x * x C] sparse latents
            image: The [N x 3 x H x W] tensor of images.
            alpha: The [N x H x W] tensor of alpha channels.
            mesh: The list of dictionaries of meshes.
            extrinsics: The [N x 4 x 4] tensor of extrinsics.
            intrinsics: The [N x 3 x 3] tensor of intrinsics.

        Returns:
            a dict with the key "loss" containing a scalar tensor.
            may also contain other keys for different terms.
        """
        reps = self.training_models['decoder'](latents)
        self.renderer.rendering_options.resolution = image.shape[-1]
        
        terms = edict(loss = 0.0, rec = 0.0)
        
        terms['reg_loss'] = sum([rep.reg_loss for rep in reps]) / len(reps)
        terms['loss'] = terms['loss'] + terms['reg_loss']
        
        geo_terms = self.geometry_losses(reps, mesh, normal_map, extrinsics, intrinsics)
        terms.update(geo_terms)
        terms['loss'] = terms['loss'] + terms['geo_loss']
                
        if self.use_color:
            color_terms = self.color_losses(reps, image, alpha, extrinsics, intrinsics)
            terms.update(color_terms)
            terms['loss'] = terms['loss'] + terms['color_loss']
             
        return terms, {}
    
    @torch.no_grad()
    def run_snapshot(
        self,
        num_samples: int,
        batch_size: int,
        verbose: bool = False,
    ) -> Dict:
        dataloader = DataLoader(
            copy.deepcopy(self.dataset),
            batch_size=batch_size,
            shuffle=True,
            num_workers=0,
            collate_fn=self.dataset.collate_fn if hasattr(self.dataset, 'collate_fn') else None,
        )

        # inference
        ret_dict = {}
        gt_images = []
        gt_normal_maps = []
        gt_meshes = []
        exts = []
        ints = []
        reps = []
        for i in range(0, num_samples, batch_size):
            batch = min(batch_size, num_samples - i)
            data = next(iter(dataloader))
            args = recursive_to_device(data, 'cuda')
            gt_images.append(args['image'] * args['alpha'][:, None])
            if self.use_color and 'normal_map' in data:
                gt_normal_maps.append(args['normal_map'])
            gt_meshes.extend(args['mesh'])
            exts.append(args['extrinsics'])
            ints.append(args['intrinsics'])
            reps.extend(self.models['decoder'](args['latents']))
        gt_images = torch.cat(gt_images, dim=0)
        ret_dict.update({f'gt_image': {'value': gt_images, 'type': 'image'}})
        if self.use_color and gt_normal_maps:
            gt_normal_maps = torch.cat(gt_normal_maps, dim=0)
            ret_dict.update({f'gt_normal_map': {'value': gt_normal_maps, 'type': 'image'}})

        # render single view
        exts = torch.cat(exts, dim=0)
        ints = torch.cat(ints, dim=0)
        self.renderer.rendering_options.bg_color = (0, 0, 0)
        self.renderer.rendering_options.resolution = gt_images.shape[-1]
        gt_render_results = self._render_batch([
            MeshExtractResult(vertices=mesh['vertices'].to(self.device), faces=mesh['faces'].to(self.device))
            for mesh in gt_meshes
        ], exts, ints, return_types=['normal'])
        ret_dict.update({f'gt_normal': {'value': self._flip_normal(gt_render_results['normal'], exts, ints), 'type': 'image'}})
        return_types = ['normal']
        if self.use_color:
            return_types.append('color')
            if 'normal_map' in data:
                return_types.append('normal_map')
        render_results = self._render_batch(reps, exts, ints, return_types=return_types)
        ret_dict.update({f'rec_normal': {'value': render_results['normal'], 'type': 'image'}})
        if 'color' in return_types:
            ret_dict.update({f'rec_image': {'value': render_results['color'], 'type': 'image'}})
        if 'normal_map' in return_types:
            ret_dict.update({f'rec_normal_map': {'value': render_results['normal_map'], 'type': 'image'}})

        # render multiview
        self.renderer.rendering_options.resolution = 512
        ## Build camera
        yaws = [0, np.pi / 2, np.pi, 3 * np.pi / 2]
        yaws_offset = np.random.uniform(-np.pi / 4, np.pi / 4)
        yaws = [y + yaws_offset for y in yaws]
        pitch = [np.random.uniform(-np.pi / 4, np.pi / 4) for _ in range(4)]

        ## render each view
        multiview_normals = []
        multiview_normal_maps = []
        miltiview_images = []
        for yaw, pitch in zip(yaws, pitch):
            orig = torch.tensor([
                np.sin(yaw) * np.cos(pitch),
                np.cos(yaw) * np.cos(pitch),
                np.sin(pitch),
            ]).float().cuda() * 2
            fov = torch.deg2rad(torch.tensor(30)).cuda()
            extrinsics = utils3d.torch.extrinsics_look_at(orig, torch.tensor([0, 0, 0]).float().cuda(), torch.tensor([0, 0, 1]).float().cuda())
            intrinsics = utils3d.torch.intrinsics_from_fov_xy(fov, fov)
            extrinsics = extrinsics.unsqueeze(0).expand(num_samples, -1, -1)
            intrinsics = intrinsics.unsqueeze(0).expand(num_samples, -1, -1)
            render_results = self._render_batch(reps, extrinsics, intrinsics, return_types=return_types)
            multiview_normals.append(render_results['normal'])
            if 'color' in return_types:
                miltiview_images.append(render_results['color'])
            if 'normal_map' in return_types:
                multiview_normal_maps.append(render_results['normal_map'])

        ## Concatenate views
        multiview_normals = torch.cat([
            torch.cat(multiview_normals[:2], dim=-2),
            torch.cat(multiview_normals[2:], dim=-2),
        ], dim=-1)
        ret_dict.update({f'multiview_normal': {'value': multiview_normals, 'type': 'image'}})
        if 'color' in return_types:
            miltiview_images = torch.cat([
                torch.cat(miltiview_images[:2], dim=-2),
                torch.cat(miltiview_images[2:], dim=-2),
            ], dim=-1)
            ret_dict.update({f'multiview_image': {'value': miltiview_images, 'type': 'image'}})
        if 'normal_map' in return_types:
            multiview_normal_maps = torch.cat([
                torch.cat(multiview_normal_maps[:2], dim=-2),
                torch.cat(multiview_normal_maps[2:], dim=-2),
            ], dim=-1)
            ret_dict.update({f'multiview_normal_map': {'value': multiview_normal_maps, 'type': 'image'}})
                            
        return ret_dict