import numpy as np from numpy import ndarray from typing import List, Union, Tuple from collections import defaultdict import os try: import open3d as o3d OPEN3D_EQUIPPED = True except: print("do not have open3d") OPEN3D_EQUIPPED = False class Exporter(): def _safe_make_dir(self, path): if os.path.dirname(path) == '': return os.makedirs(os.path.dirname(path), exist_ok=True) def _export_skeleton(self, joints: ndarray, parents: List[Union[int, None]], path: str): format = path.split('.')[-1] assert format in ['obj'] name = path.removesuffix('.obj') path = name + ".obj" self._safe_make_dir(path) J = joints.shape[0] with open(path, 'w') as file: file.write("o spring_joint\n") _joints = [] for id in range(J): pid = parents[id] if pid is None or pid == -1: continue bx, by, bz = joints[id] ex, ey, ez = joints[pid] _joints.extend([ f"v {bx} {bz} {-by}\n", f"v {ex} {ez} {-ey}\n", f"v {ex} {ez} {-ey + 0.00001}\n" ]) file.writelines(_joints) _faces = [f"f {id*3+1} {id*3+2} {id*3+3}\n" for id in range(J)] file.writelines(_faces) def _export_bones(self, bones: ndarray, path: str): format = path.split('.')[-1] assert format in ['obj'] name = path.removesuffix('.obj') path = name + ".obj" self._safe_make_dir(path) J = bones.shape[0] with open(path, 'w') as file: file.write("o bones\n") _joints = [] for bone in bones: bx, by, bz = bone[:3] ex, ey, ez = bone[3:] _joints.extend([ f"v {bx} {bz} {-by}\n", f"v {ex} {ez} {-ey}\n", f"v {ex} {ez} {-ey + 0.00001}\n" ]) file.writelines(_joints) _faces = [f"f {id*3+1} {id*3+2} {id*3+3}\n" for id in range(J)] file.writelines(_faces) def _export_skeleton_sequence(self, joints: ndarray, parents: List[Union[int, None]], path: str): format = path.split('.')[-1] assert format in ['obj'] name = path.removesuffix('.obj') path = name + ".obj" self._safe_make_dir(path) J = joints.shape[0] for i in range(J): file = open(name + f"_{i}.obj", 'w') file.write("o spring_joint\n") _joints = [] for id in range(i + 1): pid = parents[id] if pid is None: continue bx, by, bz = joints[id] ex, ey, ez = joints[pid] _joints.extend([ f"v {bx} {bz} {-by}\n", f"v {ex} {ez} {-ey}\n", f"v {ex} {ez} {-ey + 0.00001}\n" ]) file.writelines(_joints) _faces = [f"f {id*3+1} {id*3+2} {id*3+3}\n" for id in range(J)] file.writelines(_faces) file.close() def _export_mesh(self, vertices: ndarray, faces: ndarray, path: str): format = path.split('.')[-1] assert format in ['obj', 'ply'] if path.endswith('ply'): if not OPEN3D_EQUIPPED: raise RuntimeError("open3d is not available") mesh = o3d.geometry.TriangleMesh() mesh.vertices = o3d.utility.Vector3dVector(vertices) mesh.triangles = o3d.utility.Vector3iVector(faces) self._safe_make_dir(path) o3d.io.write_triangle_mesh(path, mesh) return name = path.removesuffix('.obj') path = name + ".obj" self._safe_make_dir(path) with open(path, 'w') as file: file.write("o mesh\n") _vertices = [] for co in vertices: _vertices.append(f"v {co[0]} {co[2]} {-co[1]}\n") file.writelines(_vertices) _faces = [] for face in faces: _faces.append(f"f {face[0]+1} {face[1]+1} {face[2]+1}\n") file.writelines(_faces) def _export_pc(self, vertices: ndarray, path: str, vertex_normals: Union[ndarray, None]=None, normal_size: float=0.01): if path.endswith('.ply'): if vertex_normals is not None: print("normal result will not be displayed in .ply format") name = path.removesuffix('.ply') path = name + ".ply" pc = o3d.geometry.PointCloud() pc.points = o3d.utility.Vector3dVector(vertices) # segment fault when numpy >= 2.0 !! use torch environment self._safe_make_dir(path) o3d.io.write_point_cloud(path, pc) return name = path.removesuffix('.obj') path = name + ".obj" self._safe_make_dir(path) with open(path, 'w') as file: file.write("o pc\n") _vertex = [] for co in vertices: _vertex.append(f"v {co[0]} {co[2]} {-co[1]}\n") file.writelines(_vertex) if vertex_normals is not None: new_path = path.replace('.obj', '_normal.obj') nfile = open(new_path, 'w') nfile.write("o normal\n") _normal = [] for i in range(vertices.shape[0]): co = vertices[i] x = vertex_normals[i, 0] y = vertex_normals[i, 1] z = vertex_normals[i, 2] _normal.extend([ f"v {co[0]} {co[2]} {-co[1]}\n", f"v {co[0]+0.0001} {co[2]} {-co[1]}\n", f"v {co[0]+x*normal_size} {co[2]+z*normal_size} {-(co[1]+y*normal_size)}\n", f"f {i*3+1} {i*3+2} {i*3+3}\n", ]) nfile.writelines(_normal) def _make_armature( self, vertices: Union[ndarray, None], joints: ndarray, skin: Union[ndarray, None], parents: List[Union[int, None]], names: List[str], faces: Union[ndarray, None]=None, extrude_size: float=0.03, group_per_vertex: int=-1, add_root: bool=False, do_not_normalize: bool=False, use_extrude_bone: bool=True, use_connect_unique_child: bool=True, extrude_from_parent: bool=True, tails: Union[ndarray, None]=None, ): import bpy # type: ignore from mathutils import Vector # type: ignore # make collection collection = bpy.data.collections.new('new_collection') bpy.context.scene.collection.children.link(collection) # make mesh if vertices is not None: mesh = bpy.data.meshes.new('mesh') if faces is None: faces = [] mesh.from_pydata(vertices, [], faces) mesh.update() # make object from mesh object = bpy.data.objects.new('character', mesh) # add object to scene collection collection.objects.link(object) # deselect mesh bpy.ops.object.armature_add(enter_editmode=True) armature = bpy.data.armatures.get('Armature') edit_bones = armature.edit_bones J = joints.shape[0] if tails is None: tails = joints.copy() tails[:, 2] += extrude_size connects = [False for _ in range(J)] children = defaultdict(list) for i in range(1, J): children[parents[i]].append(i) if tails is not None: if use_extrude_bone: for i in range(J): if len(children[i]) != 1 and extrude_from_parent and i != 0: pjoint = joints[parents[i]] joint = joints[i] d = joint - pjoint if np.linalg.norm(d) < 0.000001: d = np.array([0., 0., 1.]) # in case son.head == parent.head else: d = d / np.linalg.norm(d) tails[i] = joint + d * extrude_size if use_connect_unique_child: for i in range(J): if len(children[i]) == 1: child = children[i][0] tails[i] = joints[child] if parents[i] is not None and len(children[parents[i]]) == 1: connects[i] = True if add_root: bone_root = edit_bones.get('Bone') bone_root.name = 'Root' bone_root.tail = Vector((joints[0, 0], joints[0, 1], joints[0, 2])) else: bone_root = edit_bones.get('Bone') bone_root.name = names[0] bone_root.head = Vector((joints[0, 0], joints[0, 1], joints[0, 2])) bone_root.tail = Vector((joints[0, 0], joints[0, 1], joints[0, 2] + extrude_size)) def extrude_bone( edit_bones, name: str, parent_name: str, head: Tuple[float, float, float], tail: Tuple[float, float, float], connect: bool ): bone = edit_bones.new(name) bone.head = Vector((head[0], head[1], head[2])) bone.tail = Vector((tail[0], tail[1], tail[2])) bone.name = name parent_bone = edit_bones.get(parent_name) bone.parent = parent_bone bone.use_connect = connect assert not np.isnan(head).any(), f"nan found in head of bone {name}" assert not np.isnan(tail).any(), f"nan found in tail of bone {name}" for i in range(J): if add_root is False and i==0: continue edit_bones = armature.edit_bones pname = 'Root' if parents[i] is None else names[parents[i]] extrude_bone(edit_bones, names[i], pname, joints[i], tails[i], connects[i]) for i in range(J): bone = edit_bones.get(names[i]) bone.head = Vector((joints[i, 0], joints[i, 1], joints[i, 2])) bone.tail = Vector((tails[i, 0], tails[i, 1], tails[i, 2])) if vertices is None or skin is None: return # must set to object mode to enable parent_set bpy.ops.object.mode_set(mode='OBJECT') objects = bpy.data.objects for o in bpy.context.selected_objects: o.select_set(False) ob = objects['character'] arm = bpy.data.objects['Armature'] ob.select_set(True) arm.select_set(True) bpy.ops.object.parent_set(type='ARMATURE_NAME') vis = [] for x in ob.vertex_groups: vis.append(x.name) #sparsify argsorted = np.argsort(-skin, axis=1) vertex_group_reweight = skin[np.arange(skin.shape[0])[..., None], argsorted] if group_per_vertex == -1: group_per_vertex = vertex_group_reweight.shape[-1] if not do_not_normalize: vertex_group_reweight = vertex_group_reweight / vertex_group_reweight[..., :group_per_vertex].sum(axis=1)[...,None] for v, w in enumerate(skin): for ii in range(group_per_vertex): i = argsorted[v, ii] if i >= J: continue n = names[i] if n not in vis: continue ob.vertex_groups[n].add([v], vertex_group_reweight[v, ii], 'REPLACE') def _clean_bpy(self): import bpy # type: ignore for c in bpy.data.actions: bpy.data.actions.remove(c) for c in bpy.data.armatures: bpy.data.armatures.remove(c) for c in bpy.data.cameras: bpy.data.cameras.remove(c) for c in bpy.data.collections: bpy.data.collections.remove(c) for c in bpy.data.images: bpy.data.images.remove(c) for c in bpy.data.materials: bpy.data.materials.remove(c) for c in bpy.data.meshes: bpy.data.meshes.remove(c) for c in bpy.data.objects: bpy.data.objects.remove(c) for c in bpy.data.textures: bpy.data.textures.remove(c) def _export_fbx( self, path: str, vertices: Union[ndarray, None], joints: ndarray, skin: Union[ndarray, None], parents: List[Union[int, None]], names: List[str], faces: Union[ndarray, None]=None, extrude_size: float=0.03, group_per_vertex: int=-1, add_root: bool=False, do_not_normalize: bool=False, use_extrude_bone: bool=True, use_connect_unique_child: bool=True, extrude_from_parent: bool=True, tails: Union[ndarray, None]=None, ): ''' Requires bpy installed ''' import bpy # type: ignore self._safe_make_dir(path) self._clean_bpy() self._make_armature( vertices=vertices, joints=joints, skin=skin, parents=parents, names=names, faces=faces, extrude_size=extrude_size, group_per_vertex=group_per_vertex, add_root=add_root, do_not_normalize=do_not_normalize, use_extrude_bone=use_extrude_bone, use_connect_unique_child=use_connect_unique_child, extrude_from_parent=extrude_from_parent, tails=tails, ) # always enable add_leaf_bones to keep leaf bones bpy.ops.export_scene.fbx(filepath=path, check_existing=False, add_leaf_bones=False) def _export_render( self, path: str, vertices: Union[ndarray, None], faces: Union[ndarray, None], bones: Union[ndarray, None], resolution: Tuple[float, float]=[256, 256], ): import bpy # type: ignore import bpy_extras # type: ignore from mathutils import Vector # type: ignore self._safe_make_dir(path) # normalize into [-1, 1]^3 # copied from augment assert (vertices is not None) or (bones is not None) bounds = [] if vertices is not None: bounds.append(vertices) if bones is not None: bounds.append(bones[:, :3]) bounds.append(bones[:, 3:]) bounds = np.concatenate(bounds, axis=0) bound_min = bounds.min(axis=0) bound_max = bounds.max(axis=0) trans_vertex = np.eye(4) trans_vertex = _trans_to_m(-(bound_max + bound_min)/2) @ trans_vertex # scale into the cube [-1, 1] scale = np.max((bound_max - bound_min) / 2) trans_vertex = _scale_to_m(1. / scale) @ trans_vertex def _apply(v: ndarray, trans: ndarray) -> ndarray: return np.matmul(v, trans[:3, :3].transpose()) + trans[:3, 3] if vertices is not None: vertices = _apply(vertices, trans_vertex) if bones is not None: bones[:, :3] = _apply(bones[:, :3], trans_vertex) bones[:, 3:] = _apply(bones[:, 3:], trans_vertex) # bpy api calls self._clean_bpy() bpy.context.scene.render.engine = 'BLENDER_WORKBENCH' bpy.context.scene.render.film_transparent = True bpy.context.scene.display.shading.background_type = 'VIEWPORT' collection = bpy.data.collections.new('new_collection') bpy.context.scene.collection.children.link(collection) if vertices is not None: mesh_data = bpy.data.meshes.new(name="MeshData") mesh_obj = bpy.data.objects.new(name="MeshObject", object_data=mesh_data) collection.objects.link(mesh_obj) mesh_data.from_pydata((vertices).tolist(), [], faces.tolist()) mesh_data.update() def look_at(camera, point): direction = point - camera.location rot_quat = direction.to_track_quat('-Z', 'Y') camera.rotation_euler = rot_quat.to_euler() bpy.ops.object.camera_add(location=(4, -4, 2.5)) camera = bpy.context.object camera.data.angle = np.radians(25.0) look_at(camera, Vector((0, 0, -0.2))) bpy.context.scene.camera = camera bpy.context.scene.render.resolution_x = resolution[0] bpy.context.scene.render.resolution_y = resolution[1] bpy.context.scene.render.image_settings.file_format = 'PNG' bpy.context.scene.render.filepath = path bpy.ops.render.render(write_still=True) # some AI generated code to draw bones over mesh if bones is not None: # TODO: do not save image after rendering from PIL import Image, ImageDraw img_pil = Image.open(path).convert("RGBA") draw = ImageDraw.Draw(img_pil) from bpy_extras.image_utils import load_image # type: ignore bpy.context.scene.use_nodes = True nodes = bpy.context.scene.node_tree.nodes # nodes.clear() img = load_image(path) image_node = nodes.new(type='CompositorNodeImage') image_node.image = img for i, bone in enumerate(bones): head, tail = bone[:3], bone[3:] head_2d = bpy_extras.object_utils.world_to_camera_view(bpy.context.scene, camera, Vector(head)) tail_2d = bpy_extras.object_utils.world_to_camera_view(bpy.context.scene, camera, Vector(tail)) res_x, res_y = resolution head_pix = (head_2d.x * res_x, (1 - head_2d.y) * res_y) tail_pix = (tail_2d.x * res_x, (1 - tail_2d.y) * res_y) draw.line([head_pix, tail_pix], fill=(255, 0, 0, 255), width=1) img_pil.save(path) def _trans_to_m(v: ndarray): m = np.eye(4) m[0:3, 3] = v return m def _scale_to_m(r: ndarray): m = np.zeros((4, 4)) m[0, 0] = r m[1, 1] = r m[2, 2] = r m[3, 3] = 1. return m