From 022c48818002b2354cdd50980bca728be17e1b58 Mon Sep 17 00:00:00 2001 From: Hunor Laczko Date: Fri, 6 Jan 2023 11:19:34 +0000 Subject: [PATCH] tried normalization, did not work --- cloth3d/read.py | 6 +-- cloth3d/smpl_np.py | 2 + data/dataset.py | 105 ++++++++++++++++++++++++++++++++++++++------ data/render.py | 5 ++- utils/data_utils.py | 6 ++- 5 files changed, 104 insertions(+), 20 deletions(-) diff --git a/cloth3d/read.py b/cloth3d/read.py index 59f75bd..3806ff6 100644 --- a/cloth3d/read.py +++ b/cloth3d/read.py @@ -67,7 +67,7 @@ class DataReader: V -= J[0:1] # Apply rotation on z-axis zRot = zRotMatrix(info['zrot']) - return zRot.dot(V.T).T, self.smpl[gender].faces, zRot.dot(trans.T).T[None,:] + return zRot.dot(V.T).T, self.smpl[gender].faces, zRot.dot(trans.T).T[None,:], zRot.dot((J - J[0:1] + trans[None,:]).T).T[None,:], zRot.dot((J - J[0:1]).T).T[None,:] """ Garment data """ """ @@ -84,9 +84,9 @@ class DataReader: # Read garment vertices (relative to root joint) pc16_path = os.path.join(self.SRC, sample, garment + '_registered.pc16') V = readPC2Frame(pc16_path, frame, True) + # Read sample data + info = self.read_info(sample) if absolute: - # Read sample data - info = self.read_info(sample) # Transform to absolute V += info['trans'][:,frame][None] # Apply rotation on z-axis diff --git a/cloth3d/smpl_np.py b/cloth3d/smpl_np.py index e3880d7..31f2c58 100644 --- a/cloth3d/smpl_np.py +++ b/cloth3d/smpl_np.py @@ -75,6 +75,8 @@ class SMPLModel(): self.beta = beta if trans is not None: self.trans = trans + else: + self.trans = np.zeros(self.trans_shape) self.update() return self.verts, self.J diff --git a/data/dataset.py b/data/dataset.py index 53011b2..9d59411 100644 --- a/data/dataset.py +++ b/data/dataset.py @@ -45,20 +45,23 @@ class Dataset(Dataset): path = os.path.join(self.folder, frame.id) # mesh_posed = readPC2Frame(os.path.join(path, frame.garment + '_registered.pc16'), frame.frame_nr, True) mesh_posed = self.reader.read_garment_vertices(frame.id, frame.garment, frame.frame_nr) + mesh_posed_no_trans = self.reader.read_garment_vertices(frame.id, frame.garment, frame.frame_nr, absolute=False) mesh_mask = np.load(os.path.join(path, frame.garment + '_mask.npy')) # TODO: what's the difference between masking and reading .obj file faces = mask_faces(self.body_faces, mesh_mask) # faces = quads2tris(self.reader.read_garment_topology(frame.id, frame.garment)) # body = readPC2Frame(os.path.join(path, 'Body_posed.pc16'), frame.frame_nr, True) - body, body_faces, root_joint = self.reader.read_human(frame.id, frame.frame_nr) + body, body_faces, root_joint, joints, joints_no_trans = self.reader.read_human(frame.id, frame.frame_nr) + body_no_trans, _, _, _, _ = self.reader.read_human(frame.id, frame.frame_nr, absolute=False) body_faces = quads2tris(body_faces) info = self.reader.read_info(frame.id) combined = torch.Tensor(np.concatenate((body, mesh_posed[mesh_mask, :]))).to('cuda') + combined_no_trans = torch.Tensor(np.concatenate((body_no_trans, mesh_posed_no_trans[mesh_mask, :]))).to('cuda') combined_faces = torch.Tensor(np.concatenate((body_faces, faces + body.shape[0]))).to('cuda') - idx_frame = 36000 + idx_frame = 35500 x = combined[idx_frame, :].cpu().detach().numpy() y = np.array(info['camLoc'], dtype=np.float32) dist = np.sqrt(np.sum((x - y) ** 2)) @@ -71,7 +74,10 @@ class Dataset(Dataset): print(f'deodr time: {time.perf_counter() - start}') start = time.perf_counter() - depth_image, depth_mask, v2i, K, camera = self.render(combined, combined_faces, info['camLoc']) + depth_image, depth_mask, v2i, K, camera, pix_to_face = self.render(combined, combined_faces, info['camLoc']) + # cam_no_trans = info['camLoc'].copy() + # cam_no_trans[1:] = 0 + # depth_image_no_trans, depth_mask_no_trans, _, _, camera_no_trans, _ = self.render(combined_no_trans, combined_faces, cam_no_trans) print(f'pytorch3d time: {time.perf_counter() - start}') start = time.perf_counter() @@ -79,31 +85,100 @@ class Dataset(Dataset): normal_image = get_normals_from_depth(depth_image[None,:,:], depth_mask[None,:,:], K) print(f'normals time: {time.perf_counter() - start}') - root_joint_pixel = camera.transform_points_screen(torch.tensor(root_joint.astype(np.float32), device=combined.get_device()))[0,:2] + depth_mask_body = (pix_to_face <= body_faces.shape[0]) * depth_mask + depth_mask_garment = (pix_to_face > body_faces.shape[0]) * depth_mask + + + # root_joint_pixel = camera.transform_points_screen(torch.tensor(root_joint.astype(np.float32), device=combined.get_device()))[0,:2].round() + # joints_pixel = camera.transform_points_screen(torch.tensor(joints.astype(np.float32), device=combined.get_device()))[0,:,:2] + # joints_no_trans_pixel = camera_no_trans.transform_points_screen(torch.tensor(joints_no_trans.astype(np.float32), device=combined.get_device()))[0,:,:2] + # ratio = ((joints_pixel[1,:] - joints_pixel[2,:]) ** 2).sum().sqrt() / ((joints_no_trans_pixel[1,:] - joints_no_trans_pixel[2,:]) ** 2).sum().sqrt() + + # center_offset = torch.tensor([240, 320], device=root_joint_pixel.get_device()) - root_joint_pixel + # center_offset_ratio = center_offset / torch.tensor([480, 640], device=root_joint_pixel.get_device()) + # bbox = tv.ops.masks_to_boxes(depth_mask[None,:,:])[0,:] + # width = bbox[2] - bbox[0] + # height = bbox[3] - bbox[1] + # width = width + (width % 2) + # height = height + (height % 2) + # dims = torch.tensor([width, height], device=width.get_device()) + # # crop_size = torch.max(dims) + # # top = (root_joint_pixel[1] - height / 2).round().to(torch.int64) + # # left = (root_joint_pixel[0] - width / 2).round().to(torch.int64) + + # cropped = tv.transforms.functional.crop(depth_image, bbox[1].to(torch.int64), bbox[0].to(torch.int64), height.to(torch.int64), width.to(torch.int64)) + # new_root_joint_pixel = root_joint_pixel - bbox[:2] + # offset = dims / 2 - new_root_joint_pixel + # bbox[:2] = bbox[:2] - offset + # dims = dims + torch.abs(offset) + + # cropped_2 = tv.transforms.functional.crop(depth_image, bbox[1].to(torch.int64), bbox[0].to(torch.int64), dims[1].to(torch.int64), dims[0].to(torch.int64)) + + + # left = torch.tensor([0], device=bbox.get_device()) + # right = torch.tensor([0], device=bbox.get_device()) + # top = torch.tensor([0], device=bbox.get_device()) + # bottom = torch.tensor([0], device=bbox.get_device()) + # if offset[0] > 0: + # left = 2 * offset[0] + # else: + # right = -1 * 2 * offset[0] + # if offset[1] > 0: + # top = 2 * offset[1] + # else: + # bottom = -1 * 2 * offset[1] + # # centered = tv.transforms.functional.crop(cropped, new_top.to(torch.int64), new_left.to(torch.int64), new_height.to(torch.int64), new_width.to(torch.int64)) + # centered = torch.nn.functional.pad(cropped, (left.to(torch.int64), right.to(torch.int64), top.to(torch.int64), bottom.to(torch.int64)), value=-1) + # orig = depth_image[root_joint_pixel.to(torch.int64)[1], root_joint_pixel.to(torch.int64)[0]].cpu().numpy() + # new = centered[(centered.shape[0] // 2), (centered.shape[1] // 2)].cpu().numpy() + # assert new == orig, 'Center pad did not work' + + # scaled = tv.transforms.Resize((centered.shape[1] / ratio).round().int().item(), interpolation=tv.transforms.functional.InterpolationMode.NEAREST)(centered[None,None,:,:])[0,0,:,:] + # left, right = math.floor((depth_image.shape[1] - scaled.shape[1]) / 2), round((depth_image.shape[1] - scaled.shape[1]) / 2) + # top, bottom = math.floor((depth_image.shape[0] - scaled.shape[0]) / 2), round((depth_image.shape[0] - scaled.shape[0]) / 2) + # upscaled = torch.nn.functional.pad(scaled, (left, right, top, bottom), value=-1) + # Testing # pc = kornia.geometry.depth.depth_to_3d(depth_image[None,None,:,:], K, normalize_points=True) # pc = pc[0,:,depth_mask].cpu().numpy() + all_faces = combined_faces + visible_faces_idx = pix_to_face * depth_mask + visible_faces_idx = visible_faces_idx[torch.nonzero(visible_faces_idx, as_tuple=True)] + visible_vertices = torch.unique(torch.flatten(all_faces[visible_faces_idx, :])).to(torch.long) + visible_vertices_mask = (v2i[visible_vertices, 1] < depth_image.shape[0]) * (v2i[visible_vertices, 0] < depth_image.shape[1]) + visible_vertices = visible_vertices[visible_vertices_mask] + visible_pixels_mask = depth_image[v2i[visible_vertices, 1], v2i[visible_vertices, 0]] != -1 + visible_vertices = visible_vertices[visible_pixels_mask] + + x = combined[visible_vertices, :] + y = np.array(info['camLoc'], dtype=np.float32) + dist = torch.sqrt(torch.sum((x - torch.tensor(y[None,:], device=x.get_device())) ** 2, axis=-1)) + dist_img = depth_image[v2i[visible_vertices, 1], v2i[visible_vertices, 0]] + diff = torch.mean(torch.abs(dist - dist_img)) + + # combined = combined.cpu().numpy() # depth_image = depth_image.cpu().numpy() # dist_img = depth_image[v2i[idx, 1], v2i[idx, 0]] - # depth_image[v2i[idx, 1], v2i[idx, 0]] = 100 - combined = combined.cpu().numpy() + + # depth_image[v2i[visible_vertices, 1], v2i[visible_vertices, 0]] = 10 # diff = np.abs(dist - dist_img) # import plotly.express as ex - import plotly.graph_objects as go + # import plotly.graph_objects as go # ex.imshow(deodr_depth[:,:,0]).show() - # ex.imshow(depth_image).show() + # ex.imshow(depth_image.cpu().numpy()).show() # ex.imshow(normal_image[0].cpu().numpy()).show() # trans = np.array(info['trans'], dtype=np.float32)[:, self.frames[idx].frame_nr] - trans = root_joint[0,:] - go.Figure(data = [go.Scatter3d(x=combined[:, 0], y=combined[:, 1], z=combined[:, 2], marker=dict(size=3, line=dict(width=0)), mode='markers'), - # go.Scatter3d(x=[combined[idx, 0]], y=[combined[idx, 1]], z=[combined[idx, 2]], marker=dict(size=10, line=dict(width=0)), mode='markers'), - go.Scatter3d(x=[trans[0]], y=[trans[1]], z=[trans[2]], marker=dict(size=10, line=dict(width=0)), mode='markers'), - ]).show() + # root = info['camLoc'].copy() + # root[1:] = 0 + # go.Figure(data = [go.Scatter3d(x=combined[:, 0], y=combined[:, 1], z=combined[:, 2], marker=dict(size=3, line=dict(width=0)), mode='markers'), + # go.Scatter3d(x=[combined[idx, 0]], y=[combined[idx, 1]], z=[combined[idx, 2]], marker=dict(size=10, line=dict(width=0)), mode='markers'), + # go.Scatter3d(x=[root[0]], y=[root[1]], z=[root[2]], marker=dict(size=10, line=dict(width=0)), mode='markers'), + # ]).show() # go.Figure(data = [go.Scatter3d(x=pc[0,:], y=pc[1,:], z=pc[2,:], marker=dict(size=3, line=dict(width=0)), mode='markers'), # ]).show() @@ -117,7 +192,9 @@ class Dataset(Dataset): 'depth_image': depth_image, 'mesh_mask': mesh_mask, 'depth_mask': depth_mask, - 'normal_image': normal_image + 'normal_image': normal_image, + 'depth_mask_garment': depth_mask_garment, + 'depth_mask_body': depth_mask_body } diff --git a/data/render.py b/data/render.py index 4176208..9cec66c 100644 --- a/data/render.py +++ b/data/render.py @@ -184,8 +184,9 @@ class DepthRenderer(nn.Module): raster_settings=raster_settings ) output = rasterizer(Meshes(verts[None,:,:], faces[None,:,:])) - v2i = cameras.transform_points_screen(verts).cpu().numpy().astype(np.int32)[:,:2] + v2i = cameras.transform_points_screen(verts).round().to(torch.long)[:,:2] depth = output.zbuf[0,:,:,0] + pix_to_face = output.pix_to_face[0,:,:,0] mask = depth != -1 # dmin = depth.min() @@ -198,4 +199,4 @@ class DepthRenderer(nn.Module): # import plotly.express as ex # ex.imshow(depth.cpu().numpy()).show() - return depth, mask, v2i, K, cameras + return depth, mask, v2i, K, cameras, pix_to_face diff --git a/utils/data_utils.py b/utils/data_utils.py index 29092f0..91621e7 100644 --- a/utils/data_utils.py +++ b/utils/data_utils.py @@ -47,4 +47,8 @@ def get_normals_from_depth(depth_image, depth_mask, camera_intrinsic): depth_dilated = torch.permute(depth_dilated, [0,3,1,2]) normals = kornia.geometry.depth.depth_to_normals(depth_dilated, camera_intrinsic, normalize_points=True) normals = torch.permute(normals, [0,2,3,1]) * depth_mask[:,:,:,None] - return normals \ No newline at end of file + return normals + + +def center_crop_resize(img, new_center, size): + pass \ No newline at end of file