luni/reconstruction
1
0
Fork 0
Code Issues 6 Pull Requests Packages Projects 1 Releases Wiki Activity

tried normalization, did not work

Browse Source
This commit is contained in:
luni
2023-01-06 11:19:34 +00:00
parent 51b6219021
commit 022c488180
5 changed files with 104 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data/dataset.py
+91 -14
View File
@@ -45,20 +45,23 @@ class Dataset(Dataset):
path = os.path.join(self.folder, frame.id)
# mesh_posed = readPC2Frame(os.path.join(path, frame.garment + '_registered.pc16'), frame.frame_nr, True)
mesh_posed = self.reader.read_garment_vertices(frame.id, frame.garment, frame.frame_nr)
mesh_posed_no_trans = self.reader.read_garment_vertices(frame.id, frame.garment, frame.frame_nr, absolute=False)
mesh_mask = np.load(os.path.join(path, frame.garment + '_mask.npy'))
# TODO: what's the difference between masking and reading .obj file
faces = mask_faces(self.body_faces, mesh_mask)
# faces = quads2tris(self.reader.read_garment_topology(frame.id, frame.garment))
# body = readPC2Frame(os.path.join(path, 'Body_posed.pc16'), frame.frame_nr, True)
body, body_faces, root_joint = self.reader.read_human(frame.id, frame.frame_nr)
body, body_faces, root_joint, joints, joints_no_trans = self.reader.read_human(frame.id, frame.frame_nr)
body_no_trans, _, _, _, _ = self.reader.read_human(frame.id, frame.frame_nr, absolute=False)
body_faces = quads2tris(body_faces)
info = self.reader.read_info(frame.id)
combined = torch.Tensor(np.concatenate((body, mesh_posed[mesh_mask, :]))).to('cuda')
combined_no_trans = torch.Tensor(np.concatenate((body_no_trans, mesh_posed_no_trans[mesh_mask, :]))).to('cuda')
combined_faces = torch.Tensor(np.concatenate((body_faces, faces + body.shape[0]))).to('cuda')
idx_frame = 36000
idx_frame = 35500
x = combined[idx_frame, :].cpu().detach().numpy()
y = np.array(info['camLoc'], dtype=np.float32)
dist = np.sqrt(np.sum((x - y) ** 2))
@@ -71,7 +74,10 @@ class Dataset(Dataset):
print(f'deodr time: {time.perf_counter() - start}')
start = time.perf_counter()
depth_image, depth_mask, v2i, K, camera = self.render(combined, combined_faces, info['camLoc'])
depth_image, depth_mask, v2i, K, camera, pix_to_face = self.render(combined, combined_faces, info['camLoc'])
# cam_no_trans = info['camLoc'].copy()
# cam_no_trans[1:] = 0
# depth_image_no_trans, depth_mask_no_trans, _, _, camera_no_trans, _ = self.render(combined_no_trans, combined_faces, cam_no_trans)
print(f'pytorch3d time: {time.perf_counter() - start}')
start = time.perf_counter()
@@ -79,31 +85,100 @@ class Dataset(Dataset):
normal_image = get_normals_from_depth(depth_image[None,:,:], depth_mask[None,:,:], K)
print(f'normals time: {time.perf_counter() - start}')
root_joint_pixel = camera.transform_points_screen(torch.tensor(root_joint.astype(np.float32), device=combined.get_device()))[0,:2]
depth_mask_body = (pix_to_face <= body_faces.shape[0]) * depth_mask
depth_mask_garment = (pix_to_face > body_faces.shape[0]) * depth_mask
# root_joint_pixel = camera.transform_points_screen(torch.tensor(root_joint.astype(np.float32), device=combined.get_device()))[0,:2].round()
# joints_pixel = camera.transform_points_screen(torch.tensor(joints.astype(np.float32), device=combined.get_device()))[0,:,:2]
# joints_no_trans_pixel = camera_no_trans.transform_points_screen(torch.tensor(joints_no_trans.astype(np.float32), device=combined.get_device()))[0,:,:2]
# ratio = ((joints_pixel[1,:] - joints_pixel[2,:]) ** 2).sum().sqrt() / ((joints_no_trans_pixel[1,:] - joints_no_trans_pixel[2,:]) ** 2).sum().sqrt()
# center_offset = torch.tensor([240, 320], device=root_joint_pixel.get_device()) - root_joint_pixel
# center_offset_ratio = center_offset / torch.tensor([480, 640], device=root_joint_pixel.get_device())
# bbox = tv.ops.masks_to_boxes(depth_mask[None,:,:])[0,:]
# width = bbox[2] - bbox[0]
# height = bbox[3] - bbox[1]
# width = width + (width % 2)
# height = height + (height % 2)
# dims = torch.tensor([width, height], device=width.get_device())
# # crop_size = torch.max(dims)
# # top = (root_joint_pixel[1] - height / 2).round().to(torch.int64)
# # left = (root_joint_pixel[0] - width / 2).round().to(torch.int64)
# cropped = tv.transforms.functional.crop(depth_image, bbox[1].to(torch.int64), bbox[0].to(torch.int64), height.to(torch.int64), width.to(torch.int64))
# new_root_joint_pixel = root_joint_pixel - bbox[:2]
# offset = dims / 2 - new_root_joint_pixel
# bbox[:2] = bbox[:2] - offset
# dims = dims + torch.abs(offset)
# cropped_2 = tv.transforms.functional.crop(depth_image, bbox[1].to(torch.int64), bbox[0].to(torch.int64), dims[1].to(torch.int64), dims[0].to(torch.int64))
# left = torch.tensor([0], device=bbox.get_device())
# right = torch.tensor([0], device=bbox.get_device())
# top = torch.tensor([0], device=bbox.get_device())
# bottom = torch.tensor([0], device=bbox.get_device())
# if offset[0] > 0:
# left = 2 * offset[0]
# else:
# right = -1 * 2 * offset[0]
# if offset[1] > 0:
# top = 2 * offset[1]
# else:
# bottom = -1 * 2 * offset[1]
# # centered = tv.transforms.functional.crop(cropped, new_top.to(torch.int64), new_left.to(torch.int64), new_height.to(torch.int64), new_width.to(torch.int64))
# centered = torch.nn.functional.pad(cropped, (left.to(torch.int64), right.to(torch.int64), top.to(torch.int64), bottom.to(torch.int64)), value=-1)
# orig = depth_image[root_joint_pixel.to(torch.int64)[1], root_joint_pixel.to(torch.int64)[0]].cpu().numpy()
# new = centered[(centered.shape[0] // 2), (centered.shape[1] // 2)].cpu().numpy()
# assert new == orig, 'Center pad did not work'
# scaled = tv.transforms.Resize((centered.shape[1] / ratio).round().int().item(), interpolation=tv.transforms.functional.InterpolationMode.NEAREST)(centered[None,None,:,:])[0,0,:,:]
# left, right = math.floor((depth_image.shape[1] - scaled.shape[1]) / 2), round((depth_image.shape[1] - scaled.shape[1]) / 2)
# top, bottom = math.floor((depth_image.shape[0] - scaled.shape[0]) / 2), round((depth_image.shape[0] - scaled.shape[0]) / 2)
# upscaled = torch.nn.functional.pad(scaled, (left, right, top, bottom), value=-1)
# Testing
# pc = kornia.geometry.depth.depth_to_3d(depth_image[None,None,:,:], K, normalize_points=True)
# pc = pc[0,:,depth_mask].cpu().numpy()
all_faces = combined_faces
visible_faces_idx = pix_to_face * depth_mask
visible_faces_idx = visible_faces_idx[torch.nonzero(visible_faces_idx, as_tuple=True)]
visible_vertices = torch.unique(torch.flatten(all_faces[visible_faces_idx, :])).to(torch.long)
visible_vertices_mask = (v2i[visible_vertices, 1] < depth_image.shape[0]) * (v2i[visible_vertices, 0] < depth_image.shape[1])
visible_vertices = visible_vertices[visible_vertices_mask]
visible_pixels_mask = depth_image[v2i[visible_vertices, 1], v2i[visible_vertices, 0]] != -1
visible_vertices = visible_vertices[visible_pixels_mask]
x = combined[visible_vertices, :]
y = np.array(info['camLoc'], dtype=np.float32)
dist = torch.sqrt(torch.sum((x - torch.tensor(y[None,:], device=x.get_device())) ** 2, axis=-1))
dist_img = depth_image[v2i[visible_vertices, 1], v2i[visible_vertices, 0]]
diff = torch.mean(torch.abs(dist - dist_img))
# combined = combined.cpu().numpy()
# depth_image = depth_image.cpu().numpy()
# dist_img = depth_image[v2i[idx, 1], v2i[idx, 0]]
# depth_image[v2i[idx, 1], v2i[idx, 0]] = 100
combined = combined.cpu().numpy()
# depth_image[v2i[visible_vertices, 1], v2i[visible_vertices, 0]] = 10
# diff = np.abs(dist - dist_img)
# import plotly.express as ex
import plotly.graph_objects as go
# import plotly.graph_objects as go
# ex.imshow(deodr_depth[:,:,0]).show()
# ex.imshow(depth_image).show()
# ex.imshow(depth_image.cpu().numpy()).show()
# ex.imshow(normal_image[0].cpu().numpy()).show()
# trans = np.array(info['trans'], dtype=np.float32)[:, self.frames[idx].frame_nr]
trans = root_joint[0,:]
go.Figure(data = [go.Scatter3d(x=combined[:, 0], y=combined[:, 1], z=combined[:, 2], marker=dict(size=3, line=dict(width=0)), mode='markers'),
# go.Scatter3d(x=[combined[idx, 0]], y=[combined[idx, 1]], z=[combined[idx, 2]], marker=dict(size=10, line=dict(width=0)), mode='markers'),
go.Scatter3d(x=[trans[0]], y=[trans[1]], z=[trans[2]], marker=dict(size=10, line=dict(width=0)), mode='markers'),
]).show()
# root = info['camLoc'].copy()
# root[1:] = 0
# go.Figure(data = [go.Scatter3d(x=combined[:, 0], y=combined[:, 1], z=combined[:, 2], marker=dict(size=3, line=dict(width=0)), mode='markers'),
# go.Scatter3d(x=[combined[idx, 0]], y=[combined[idx, 1]], z=[combined[idx, 2]], marker=dict(size=10, line=dict(width=0)), mode='markers'),
# go.Scatter3d(x=[root[0]], y=[root[1]], z=[root[2]], marker=dict(size=10, line=dict(width=0)), mode='markers'),
# ]).show()
# go.Figure(data = [go.Scatter3d(x=pc[0,:], y=pc[1,:], z=pc[2,:], marker=dict(size=3, line=dict(width=0)), mode='markers'),
# ]).show()
@@ -117,7 +192,9 @@ class Dataset(Dataset):
'depth_image': depth_image,
'mesh_mask': mesh_mask,
'depth_mask': depth_mask,
'normal_image': normal_image
'normal_image': normal_image,
'depth_mask_garment': depth_mask_garment,
'depth_mask_body': depth_mask_body
}
data/render.py
+3 -2
View File
@@ -184,8 +184,9 @@ class DepthRenderer(nn.Module):
raster_settings=raster_settings
)
output = rasterizer(Meshes(verts[None,:,:], faces[None,:,:]))
v2i = cameras.transform_points_screen(verts).cpu().numpy().astype(np.int32)[:,:2]
v2i = cameras.transform_points_screen(verts).round().to(torch.long)[:,:2]
depth = output.zbuf[0,:,:,0]
pix_to_face = output.pix_to_face[0,:,:,0]
mask = depth != -1
# dmin = depth.min()
@@ -198,4 +199,4 @@ class DepthRenderer(nn.Module):
# import plotly.express as ex
# ex.imshow(depth.cpu().numpy()).show()
return depth, mask, v2i, K, cameras
return depth, mask, v2i, K, cameras, pix_to_face