From 0652130e652656e71e91d3ac04be2a2328f1886e Mon Sep 17 00:00:00 2001 From: Hunor Laczko Date: Wed, 28 Dec 2022 16:10:54 +0000 Subject: [PATCH] working depth render --- data/dataset.py | 35 +++++++++++++++++++++++++++++++++-- data/render.py | 39 ++++++++++++++++++++++++++++++--------- 2 files changed, 63 insertions(+), 11 deletions(-) diff --git a/data/dataset.py b/data/dataset.py index c51923d..df95b67 100644 --- a/data/dataset.py +++ b/data/dataset.py @@ -38,7 +38,7 @@ class Dataset(Dataset): return len(self.frames) - # TODO: make it work kwith two part outfits + # TODO: make it work with two part outfits def __getitem__(self, idx): frame : Frame = self.frames[idx] path = os.path.join(self.folder, frame.id) @@ -46,6 +46,7 @@ class Dataset(Dataset): mesh_posed = self.reader.read_garment_vertices(frame.id, frame.garment, frame.frame_nr) mask = np.load(os.path.join(path, frame.garment + '_mask.npy')) + # TODO: what's the difference between masking and reading .obj file faces = mask_faces(self.body_faces, mask) # faces = quads2tris(self.reader.read_garment_topology(frame.id, frame.garment)) # body = readPC2Frame(os.path.join(path, 'Body_posed.pc16'), frame.frame_nr, True) @@ -55,14 +56,44 @@ class Dataset(Dataset): info = self.reader.read_info(frame.id) combined = torch.Tensor(np.concatenate((body, mesh_posed[mask, :]))).to('cuda') combined_faces = torch.Tensor(np.concatenate((body_faces, faces + body.shape[0]))).to('cuda') + combined_faces_int = torch.Tensor(np.concatenate((body_faces, faces + body.shape[0])).astype(np.int32)).to('cuda') + + idx = 36000 + x = combined[idx, :].cpu().detach().numpy() + y = np.array(info['camLoc'], dtype=np.float32) + dist = np.sqrt(np.sum((x - y) ** 2)) start = time.perf_counter() # depth_image = self.render.render(mesh_posed[mask, :], faces, info['camLoc'][0])[:,:,0] # depth_image = self.render.render(combined, combined_faces, info['camLoc'][0])[:,:,0] deodr_depth = self.deodr_render.render(cam=info['camLoc'].astype(np.float32), mesh=combined.cpu().numpy(), faces=combined_faces.cpu().numpy().astype(np.int32)) - depth_image = self.render(combined, combined_faces, info['camLoc']) + # deodr_depth = self.deodr_render.render(cam=info['camLoc'].astype(np.float32), mesh=combined, faces=combined_faces.cpu().numpy().astype(np.int32)) t = time.perf_counter() - start print(t) + start = time.perf_counter() + depth_image, v2i = self.render(combined, combined_faces, info['camLoc']) + t = time.perf_counter() - start + print(t) + + depth_image = depth_image.cpu().numpy() + dist_img = depth_image[v2i[idx, 1], v2i[idx, 0]] + depth_image[v2i[idx, 1], v2i[idx, 0]] = 100 + combined = combined.cpu().numpy() + + diff = np.abs(dist - dist_img) + + import plotly.express as ex + import plotly.graph_objects as go + ex.imshow(deodr_depth[:,:,0]).show() + ex.imshow(depth_image).show() + + go.Figure(data = [go.Scatter3d(x=combined[:, 0], y=combined[:, 1], z=combined[:, 2], marker=dict(size=3, line=dict(width=0)), mode='markers'), + go.Scatter3d(x=[combined[idx, 0]], y=[combined[idx, 1]], z=[combined[idx, 2]], marker=dict(size=10, line=dict(width=0)), mode='markers'), + ]).show() + + # v2i_coverage = np.zeros((480, 640), dtype=np.float32) + # v2i_coverage[v2i[:,0], v2i[:, 1]] = 1 + # ex.imshow(v2i_coverage).show() # image = self.transforms(image) return { 'mesh_posed': mesh_posed, 'depth_image': depth_image, 'mask': mask } diff --git a/data/render.py b/data/render.py index 649c397..b561c6d 100644 --- a/data/render.py +++ b/data/render.py @@ -1,15 +1,22 @@ +import os +import sys +import torch import numpy as np - from deodr import Camera, Scene3D from deodr.triangulated_mesh import TriMesh +from deodr.pytorch.triangulated_mesh_pytorch import ColoredTriMeshPytorch +from deodr.pytorch.differentiable_renderer_pytorch import Scene3DPytorch, CameraPytorch + +sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/..') +from cloth3d.util import proj class RenderCore: def __init__(self, max_depth=0, depth_scale=1): self.mesh = None self.depthScale = depth_scale - self.scene = Scene3D(0) + self.scene = Scene3D() self.set_max_depth(max_depth) def set_mesh(self, vertices, faces): @@ -141,6 +148,8 @@ from pytorch3d.renderer import ( TexturesVertex ) from pytorch3d.structures import Meshes +from pytorch3d.utils import cameras_from_opencv_projection + class DepthRenderer(nn.Module): def __init__(self) -> None: @@ -149,13 +158,21 @@ class DepthRenderer(nn.Module): def forward(self, verts, faces, cam): - R, T = look_at_view_transform(2.7, 0, 0) + # R, T = look_at_view_transform(2.7, 0, 0) # cameras = FoVPerspectiveCameras(device='cuda', R=R, T=T) - ext = extrinsic(cam) - cameras = PerspectiveCameras(device='cuda', R=ext[None, :, :3], T=ext[None, :, 3]) + + ext = torch.tensor(extrinsic(cam).astype(np.float32), device='cuda') + # ext = proj(cam) + fp = np.array([240 * (480 / 36), 320 * (640 / 36)], dtype=np.float32) + img_size = np.array([480, 640], dtype=np.float32) + # K = np.eye(4, dtype=np.float32) + # K[:3,:3] = intrinsic() + # cameras = PerspectiveCameras(device='cuda', R=ext[None, :, :3], T=ext[None, :, 3], focal_length=1,)# principal_point=fp[None, :], image_size=img_size[None, :]) + + cameras = cameras_from_opencv_projection(R=ext[None, :, :3], tvec=ext[None, :, 3], camera_matrix=torch.tensor(intrinsic()[None, :, :], device='cuda'), image_size=torch.tensor(img_size[None, :], device='cuda')) raster_settings = RasterizationSettings( - image_size=512, + image_size=(480, 640), blur_radius=0.0, faces_per_pixel=1, bin_size=0 @@ -166,14 +183,18 @@ class DepthRenderer(nn.Module): raster_settings=raster_settings ) output = rasterizer(Meshes(verts[None,:,:], faces[None,:,:])) + v2i = cameras.transform_points_screen(verts).cpu().numpy().astype(np.int32)[:,:2] depth = output.zbuf[0,:,:,0] mask = depth != -1 # dmin = depth.min() dmean = depth[mask].mean() # dmax = depth[mask].max() - depth[mask] = depth[mask] - dmean - depth[~mask] = 0.0 + # depth[mask] = depth[mask] - dmean + # depth[~mask] = 0.0 # mask = mask.astype(np.float32) - return depth + # import plotly.express as ex + # ex.imshow(depth.cpu().numpy()).show() + + return depth, v2i