working depth render

data/dataset.py

@@ -38,7 +38,7 @@ class Dataset(Dataset):
         return len(self.frames)
 
 
-    # TODO: make it work kwith two part outfits 
+    # TODO: make it work with two part outfits 
     def __getitem__(self, idx):
         frame : Frame = self.frames[idx]
         path = os.path.join(self.folder, frame.id)
@@ -46,6 +46,7 @@ class Dataset(Dataset):
         mesh_posed = self.reader.read_garment_vertices(frame.id, frame.garment, frame.frame_nr)
 
         mask = np.load(os.path.join(path, frame.garment + '_mask.npy'))
+        # TODO: what's the difference between masking and reading .obj file
         faces = mask_faces(self.body_faces, mask)
         # faces = quads2tris(self.reader.read_garment_topology(frame.id, frame.garment))
         # body = readPC2Frame(os.path.join(path, 'Body_posed.pc16'), frame.frame_nr, True)
@@ -55,14 +56,44 @@ class Dataset(Dataset):
         info = self.reader.read_info(frame.id)
         combined = torch.Tensor(np.concatenate((body, mesh_posed[mask, :]))).to('cuda')
         combined_faces = torch.Tensor(np.concatenate((body_faces, faces + body.shape[0]))).to('cuda')
+        combined_faces_int = torch.Tensor(np.concatenate((body_faces, faces + body.shape[0])).astype(np.int32)).to('cuda')
+
+        idx = 36000
+        x = combined[idx, :].cpu().detach().numpy()
+        y = np.array(info['camLoc'], dtype=np.float32)
+        dist = np.sqrt(np.sum((x - y) ** 2))
 
         start = time.perf_counter()
         # depth_image = self.render.render(mesh_posed[mask, :], faces, info['camLoc'][0])[:,:,0]
         # depth_image = self.render.render(combined, combined_faces, info['camLoc'][0])[:,:,0]
         deodr_depth = self.deodr_render.render(cam=info['camLoc'].astype(np.float32), mesh=combined.cpu().numpy(), faces=combined_faces.cpu().numpy().astype(np.int32))
-        depth_image = self.render(combined, combined_faces, info['camLoc'])
+        # deodr_depth = self.deodr_render.render(cam=info['camLoc'].astype(np.float32), mesh=combined, faces=combined_faces.cpu().numpy().astype(np.int32))
         t = time.perf_counter() - start
         print(t)
+        start = time.perf_counter()
+        depth_image, v2i = self.render(combined, combined_faces, info['camLoc'])
+        t = time.perf_counter() - start
+        print(t)
+
+        depth_image = depth_image.cpu().numpy()
+        dist_img = depth_image[v2i[idx, 1], v2i[idx, 0]]
+        depth_image[v2i[idx, 1], v2i[idx, 0]] = 100
+        combined = combined.cpu().numpy()
+
+        diff = np.abs(dist - dist_img)
+
+        import plotly.express as ex
+        import plotly.graph_objects as go
+        ex.imshow(deodr_depth[:,:,0]).show()
+        ex.imshow(depth_image).show()
+
+        go.Figure(data = [go.Scatter3d(x=combined[:, 0], y=combined[:, 1], z=combined[:, 2], marker=dict(size=3, line=dict(width=0)), mode='markers'),
+                          go.Scatter3d(x=[combined[idx, 0]], y=[combined[idx, 1]], z=[combined[idx, 2]], marker=dict(size=10, line=dict(width=0)), mode='markers'),
+                         ]).show()
+
+        # v2i_coverage = np.zeros((480, 640), dtype=np.float32)
+        # v2i_coverage[v2i[:,0], v2i[:, 1]] = 1
+        # ex.imshow(v2i_coverage).show()
 
         # image = self.transforms(image)
         return { 'mesh_posed': mesh_posed, 'depth_image': depth_image, 'mask': mask }

data/render.py

@@ -1,15 +1,22 @@
+import os
+import sys
+import torch
 import numpy as np
 
-
 from deodr import Camera, Scene3D
 from deodr.triangulated_mesh import TriMesh
+from deodr.pytorch.triangulated_mesh_pytorch import ColoredTriMeshPytorch
+from deodr.pytorch.differentiable_renderer_pytorch import Scene3DPytorch, CameraPytorch
+
+sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/..')
+from cloth3d.util import proj
 
 
 class RenderCore:
     def __init__(self, max_depth=0, depth_scale=1):
         self.mesh = None
         self.depthScale = depth_scale
-        self.scene = Scene3D(0)
+        self.scene = Scene3D()
         self.set_max_depth(max_depth)
 		
     def set_mesh(self, vertices, faces):
@@ -141,6 +148,8 @@ from pytorch3d.renderer import (
     TexturesVertex
 )
 from pytorch3d.structures import Meshes
+from pytorch3d.utils import cameras_from_opencv_projection
+
 
 class DepthRenderer(nn.Module):
     def __init__(self) -> None:
@@ -149,13 +158,21 @@ class DepthRenderer(nn.Module):
 
 
     def forward(self, verts, faces, cam):
-        R, T = look_at_view_transform(2.7, 0, 0) 
+        # R, T = look_at_view_transform(2.7, 0, 0) 
         # cameras = FoVPerspectiveCameras(device='cuda', R=R, T=T)
-        ext = extrinsic(cam)
+
-        cameras = PerspectiveCameras(device='cuda', R=ext[None, :, :3], T=ext[None, :, 3])
+        ext = torch.tensor(extrinsic(cam).astype(np.float32), device='cuda')
+        # ext = proj(cam)
+        fp = np.array([240 * (480 / 36), 320 * (640 / 36)], dtype=np.float32)
+        img_size = np.array([480, 640], dtype=np.float32)
+        # K = np.eye(4, dtype=np.float32)
+        # K[:3,:3] = intrinsic()
+        # cameras = PerspectiveCameras(device='cuda', R=ext[None, :, :3], T=ext[None, :, 3], focal_length=1,)# principal_point=fp[None, :], image_size=img_size[None, :])
+
+        cameras = cameras_from_opencv_projection(R=ext[None, :, :3], tvec=ext[None, :, 3], camera_matrix=torch.tensor(intrinsic()[None, :, :], device='cuda'), image_size=torch.tensor(img_size[None, :], device='cuda'))
 
         raster_settings = RasterizationSettings(
-                            image_size=512, 
+                            image_size=(480, 640), 
                             blur_radius=0.0, 
                             faces_per_pixel=1, 
                             bin_size=0
@@ -166,14 +183,18 @@ class DepthRenderer(nn.Module):
                     raster_settings=raster_settings
                 )
         output = rasterizer(Meshes(verts[None,:,:], faces[None,:,:]))
+        v2i = cameras.transform_points_screen(verts).cpu().numpy().astype(np.int32)[:,:2]
         depth = output.zbuf[0,:,:,0]
 
         mask = depth != -1
         # dmin = depth.min()
         dmean = depth[mask].mean()
         # dmax = depth[mask].max()
-        depth[mask] = depth[mask] - dmean
+        # depth[mask] = depth[mask] - dmean
-        depth[~mask] = 0.0
+        # depth[~mask] = 0.0
         # mask = mask.astype(np.float32)
 
-        return depth
+        # import plotly.express as ex
+        # ex.imshow(depth.cpu().numpy()).show()
+
+        return depth, v2i