luni/cvpr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

commit all

Browse Source
This commit is contained in:
luni
2023-01-25 12:25:59 +00:00
parent 1bf224841f
commit 7470ca9fb6
139 changed files with 82069 additions and 811 deletions
Download Patch File Download Diff File Expand all files Collapse all files
custom_utils/create_uv_map.py
+596
View File
@@ -0,0 +1,596 @@
import os
import sys
import shutil
import tqdm
import random
import math
import collections
import tps
import numpy as np
import scipy as sp
import trimesh as tm
import cmapy as cm
import scipy.io as sio
import matplotlib.pyplot as plt
import plotly.express as ex
def m2uv_array(m2uv, is_smpl_orig=False):
result = np.zeros((14475 if not is_smpl_orig else 3638, 2), dtype=np.int32)
for k, v in m2uv.items():
if len(v) == 2:
result[k] = list(v)
elif len(v) == 1:
x = v.pop()
result[k] = [x, x]
else:
assert False
return result
def get_body():
smpl = SMPL('FeatureLines/superSMPL')
pose = np.zeros((1, 72), dtype=np.float32)
beta = np.zeros((1, 11), dtype=np.float32)
R = smpl.run_rodrigues(pose)
T, _, _ = smpl.run(R, beta, True)
return np.squeeze(T)
def plot_3d(x, y=None):
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(111, projection='3d')
ax.set_xlim3d(-1, 1)
ax.set_ylim3d(-1, 1)
ax.set_zlim3d(-1, 1)
ax.set_xlabel("X")
ax.set_ylabel("Y")
ax.set_zlabel("Z")
ax.scatter(x[:, 0], x[:, 1], x[:, 2], s=5)
if y is not None:
ax.scatter(y[:, 0], y[:, 1], y[:, 2], s=5)
plt.show()
def plot_uv(x, y=None):
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(111)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.scatter(x[:, 0], x[:, 1])
if y is not None:
ax.scatter(y[:, 0], y[:, 1])
plt.show()
"""
Mesh to UV map
Computes correspondences between 3D mesh and UV map
NOTE: 3D mesh vertices can have multiple correspondences with UV vertices
"""
def mesh2UV(F, Ft):
m2uv = {v: set() for f in F for v in f}
for f, ft in zip(F, Ft):
for v, vt in zip(f, ft):
m2uv[v].add(vt)
# m2uv = {k:list(v) for k,v in m2uv.items()}
return m2uv
def filter_m2u(m2u, to_keep):
for key in m2u:
if len(m2u[key]) == 2:
v = list(m2u[key])
if v[0] in to_keep:
m2u[key].remove(v[1])
else:
m2u[key].remove(v[0])
return m2u
def convert_to_numbers(m2u):
for key in m2u:
m2u[key] = m2u[key].pop()
return m2u
def get_masked_index(mask, index):
m = np.zeros((mask.shape[0]), dtype=bool)
m[index] = True
m = m[mask]
new_index = np.argmax(m)
return new_index
def get_closest_point(verts_1, verts_2):
dist = sp.spatial.distance.cdist(verts_1, verts_2)
closest = np.argmin(dist, axis=-1)
return closest
def get_normal_edge(p1, p2):
d = p2 - p1
return np.array([-d[1], d[0]])
def get_normal_vertex(p1, p2, p3):
n1 = get_normal_edge(p1, p2)
n2 = get_normal_edge(p2, p3)
n = (n1 + n2)
n = n / np.linalg.norm(n) / 20
return n
def get_distance(p1, p2, points):
d = np.array([np.linalg.norm(np.cross(p2 - p1, point - p1)) / np.linalg.norm(p2 - p1) for point in points])
return d
def get_normals(verts):
v = verts.copy()
v = np.concatenate([v[-1][np.newaxis, :], v, v[0][np.newaxis, :]])
normals = np.zeros_like(verts)
for i in range(1, v.shape[0] - 1):
normals[i - 1] = get_normal_vertex(v[i - 1], v[i], v[i + 1])
return normals
def get_closest_points_normals(verts1, verts2):
dist_2 = sp.spatial.distance.cdist(verts1, verts2)
normals = get_normals(verts1)
# fig = plt.figure(figsize=(15, 15))
# ax = fig.add_subplot(111)
# ax.set_xlim(0, 1)
# ax.set_ylim(0, 1)
# ax.set_aspect('equal')
# ax.scatter(verts1[:, 0], verts1[:, 1])
# for p1, p2 in zip(verts1, verts1 + normals):
# ax.plot([p1[0], p2[0]], [p1[1], p2[1]])
# plt.show()
closest = []
i = 0
for p1, p2 in zip(verts1, verts1 + normals):
dist = get_distance(p1, p2, verts2)
closest.append(np.argmin(dist + dist_2[i, :]))
i += 1
return np.array(closest)
def back():
# load data
data_body = np.load("Blender/uv_data_1.npz", allow_pickle=True)
data_back = np.load("Blender/uv_data_plane_1_2.npz", allow_pickle=True)
# translate back uv to match body
uv_back = data_back.f.Vt
# uv_back[:, 0] = uv_back[:, 0] + 0.1
# uv_back[:, 1] = uv_back[:, 1] + 0.42
# find front and back indices in uv
uv_verts = np.append(data_body.f.Vt, np.zeros((data_body.f.Vt.shape[0], 1)), axis=-1)
m = tm.Trimesh(uv_verts, data_body.f.Ft, process=False, validate=False)
front, back = tm.graph.connected_components(m.edges, min_len=1, nodes=None, engine=None)
# get mesh to uv mapping for the back vertices
m2u_body = mesh2UV(data_body.f.F, data_body.f.Ft)
m2u_body = filter_m2u(m2u_body, set(back))
m2u_body = convert_to_numbers(m2u_body)
m2u_back = mesh2UV(data_back.f.F, data_back.f.Ft)
m2u_back = convert_to_numbers(m2u_back)
# create mask for filtering vertices belonging to the back
body_back_mask = np.zeros((data_body.f.Vt.shape[0]), dtype=bool)
body_back_mask[back] = True
# define which points should match after the transform on the mesh
matching_points = np.array([[3284, 3], [9557, 13], [9278, 15], [8095, 1], [10597, 10], [10602, 17], [7114, 9], [6989, 16], [6983, 11], [4219, 0], [5628, 14], [5908, 12], [1447, 2], [1086, 5], [21, 8], [33, 4], [1888, 7], [2931, 6]])
# map matching point to uv coordinates
for i in range(matching_points.shape[0]):
matching_points[i, :] = [get_masked_index(body_back_mask, m2u_body[matching_points[i,0]]), m2u_back[matching_points[i, 1]]]
x = data_body.f.Vt[body_back_mask, :]
y = uv_back
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(121)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.set_aspect('equal')
ax.scatter(x[:, 0], x[:, 1])
if y is not None:
ax.scatter(y[:, 0], y[:, 1])
for pair in matching_points:
ax.scatter([x[pair[0], 0], y[pair[1], 0]], [x[pair[0], 1], y[pair[1], 1]], s=50)
# get transformation from body to back contour
source = x[matching_points[:, 0], :]
destination = y[matching_points[:, 1], :]
trans = tps.TPS(source, destination)
transformed = trans(data_body.f.Vt[back, :])
ax = fig.add_subplot(122)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.set_aspect('equal')
ax.scatter(transformed[:, 0], transformed[:, 1])
if y is not None:
ax.scatter(y[:, 0], y[:, 1])
for pair in matching_points:
ax.scatter([transformed[pair[0], 0], y[pair[1], 0]], [transformed[pair[0], 1], y[pair[1], 1]], s=50)
plt.show()
data_back_high = np.load("Blender/uv_data_plane_1_high_res.npz", allow_pickle=True)
y_high = data_back_high.f.Vt
# y_high[:, 0] = y_high[:, 0] + 0.1
# y_high[:, 1] = y_high[:, 1] + 0.42
closest_points = get_closest_points_normals(y_high, transformed)
# closest_points = get_closest_point(y_high, transformed)
unique_mask = np.zeros_like(closest_points, dtype=bool)
unique_mask[np.unique(closest_points, return_index=True)[1]] = True
matching_points = np.array([closest_points[unique_mask], np.array(range(y_high.shape[0]))[unique_mask]]).T
# matching_points_2 = []
# for i, p in enumerate(matching_points):
# if i % 10 == 0:
# matching_points_2.append(p)
# matching_points = np.array(matching_points_2)
source_2 = transformed[matching_points[:, 0], :]
destination_2 = y_high[matching_points[:, 1], :]
trans_2 = tps.TPS(source_2, destination_2)
transformed_2 = trans_2(transformed)
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(121)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.set_aspect('equal')
ax.scatter(source_2[:, 0], source_2[:, 1])
ax.scatter(destination_2[:, 0], destination_2[:, 1])
for i, pair in enumerate(zip(source_2, destination_2)):
if i % 5 == 0:
s, d = pair
ax.scatter([s[0], d[0]], [s[1], d[1]])
ax = fig.add_subplot(122)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.set_aspect('equal')
ax.scatter(transformed_2[:, 0], transformed_2[:, 1])
ax.scatter(y_high[:, 0], y_high[:, 1])
plt.show()
return transformed_2
def match_back_to_front(front_uv):
# load data
data_body = np.load("Blender/uv_data_smpl_2.npz", allow_pickle=True)
# data_back = np.load("Blender/uv_data_plane_1_2.npz", allow_pickle=True)
# translate back uv to match body
# uv_back = data_back.f.Vt
# uv_back[:, 0] = uv_back[:, 0] + 0.1
# uv_back[:, 1] = uv_back[:, 1] + 0.42
# find front and back indices in uv
uv_verts = np.append(data_body.f.Vt, np.zeros((data_body.f.Vt.shape[0], 1)), axis=-1)
m = tm.Trimesh(uv_verts, data_body.f.Ft, process=False, validate=False)
front, back = tm.graph.connected_components(m.edges, min_len=1, nodes=None, engine=None)
# get mesh to uv mapping for the back vertices
m2u_body = mesh2UV(data_body.f.F, data_body.f.Ft)
m2u_body = filter_m2u(m2u_body, set(back))
m2u_body = convert_to_numbers(m2u_body)
# m2u_back = mesh2UV(data_back.f.F, data_back.f.Ft)
# m2u_back = convert_to_numbers(m2u_back)
# create mask for filtering vertices belonging to the back
body_back_mask = np.zeros((data_body.f.Vt.shape[0]), dtype=bool)
body_back_mask[back] = True
body_front_mask = np.zeros((data_body.f.Vt.shape[0]), dtype=bool)
body_front_mask[front] = True
# define which points should match after the transform on the mesh
# matching_points = np.array([[3284, 3], [9557, 13], [9278, 15], [8095, 1], [10597, 10], [10602, 17], [7114, 9], [6989, 16], [6983, 11], [4219, 0], [5628, 14], [5908, 12], [1447, 2], [1086, 5], [21, 8], [33, 4], [1888, 7], [2931, 6]])
# # map matching point to uv coordinates
# for i in range(matching_points.shape[0]):
# matching_points[i, :] = [get_masked_index(body_back_mask, m2u_body[matching_points[i,0]]), m2u_back[matching_points[i, 1]]]
m2u_body_all = mesh2UV(data_body.f.F, data_body.f.Ft)
m2u_body_all = m2uv_array(m2u_body_all)
border_pairs = m2u_body_all[:, 0] != m2u_body_all[:, 1]
border_pairs = m2u_body_all[border_pairs, :]
matching_points = border_pairs
matching_points_sorted = []
for p in matching_points:
if p[0] in front:
matching_points_sorted.append([p[0], p[1]])
elif p[0] in back:
matching_points_sorted.append([p[1], p[0]])
else:
assert False
# add neck V shape control point
matching_points_sorted.append([m2u_body_all[10228][0], m2u_body_all[33][0]])
matching_points = np.array(matching_points_sorted)
mm = matching_points.copy()
for i in range(matching_points.shape[0]):
matching_points[i, :] = [get_masked_index(body_back_mask, matching_points[i,1]), get_masked_index(body_front_mask, matching_points[i,0])]
temp = np.zeros_like(data_body.f.Vt)
temp[front, :] = front_uv
x = data_body.f.Vt[body_back_mask, :]
y = temp[body_front_mask, :] # front_uv #
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(121)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.set_aspect('equal')
ax.scatter(x[:, 0], x[:, 1])
if y is not None:
ax.scatter(y[:, 0], y[:, 1])
for pair in matching_points:
ax.scatter([x[pair[0], 0], y[pair[1], 0]], [x[pair[0], 1], y[pair[1], 1]], s=50)
# get transformation from body to back contour
source = x[matching_points[:, 0], :]
destination = y[matching_points[:, 1], :]
trans = tps.TPS(source, destination)
transformed = trans(data_body.f.Vt[back, :])
ax = fig.add_subplot(122)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.set_aspect('equal')
ax.scatter(transformed[:, 0], transformed[:, 1])
if y is not None:
ax.scatter(y[:, 0], y[:, 1])
for pair in matching_points:
ax.scatter([transformed[pair[0], 0], y[pair[1], 0]], [transformed[pair[0], 1], y[pair[1], 1]], s=50)
plt.show()
return transformed
def generate_mask():
import cv2
res = 1024
front = np.load("Blender/uv_data_plane_0_high_res_2.npz", allow_pickle=True)
back = np.load("Blender/uv_data_plane_1_high_res_2.npz", allow_pickle=True)
points = np.round(np.concatenate((front.f.Vt, back.f.Vt)) * res).astype(np.int32)
img = np.zeros((res, res, 3))
img[points[:, 0], points[:, 1], :] = 255
cv2.imwrite('MeshUtils/mask_1024.png', img)
# do paint now
img = cv2.imread('MeshUtils/mask_1024_2.png')
img = img[:, :, 0] == 255
np.save('MeshUtils/mask_1024.npy', img)
def generate_mask_2():
import cv2
res = 256
front = np.load("Blender/uv_data_plane_stretched_front_high_extra.npz", allow_pickle=True)
back = np.load("Blender/uv_data_plane_stretched_front_high_extra.npz", allow_pickle=True)
points = np.floor(np.concatenate((front.f.Vt, back.f.Vt + [1, 0])) * res).astype(np.int32)
img = np.zeros((res * 2, res, 3))
img[points[:, 0], points[:, 1], :] = 255
cv2.imwrite('MeshUtils/mask_256x512.png', img)
# do paint now
img = cv2.imread('MeshUtils/mask_256x512.png')
img = img[:, :, 0] > 240
np.save('MeshUtils/mask_256x512.npy', img)
pass
def generate_uv_mask_border():
import cv2
mask = np.load('MeshUtils/mask_256x512.npy')
uv_res = 256
mask_border = (cv2.resize(mask * 255.0, (uv_res, uv_res * 2)) / 255).astype(bool)[:, :, np.newaxis]
mask_border = cv2.erode(mask_border.astype(np.float32), cv2.getStructuringElement(cv2.MORPH_RECT, (2 * 2 + 1, 2 * 2 + 1), (2, 2)), iterations=1).astype(bool)
mask[mask_border == True] = 0
# plt.imshow(mask)
idx_front = np.transpose(np.vstack(np.nonzero(mask[:256, :])), [1,0])
test = np.zeros_like(mask)
test[idx_front[:,0], idx_front[:,1]] = 1
idx_back = idx_front.copy()
idx_back[:, 0] += 256
test[idx_back[:,0], idx_back[:,1]] = 1
pass
def front():
# load data
data_body = np.load("custom_data/uv_map/uv_data_smpl_2.npz", allow_pickle=True)
data_front = np.load("custom_data/uv_map/uv_data_plane_stretched_front_low.npz", allow_pickle=True)
# translate back uv to match body
uv_front = data_front.f.Vt
# uv_back[:, 0] = uv_back[:, 0] + 0.1
# uv_back[:, 1] = uv_back[:, 1] + 0.42
# find front and back indices in uv
uv_verts = np.append(data_body.f.Vt, np.zeros((data_body.f.Vt.shape[0], 1)), axis=-1)
m = tm.Trimesh(uv_verts, data_body.f.Ft, process=False, validate=False)
front, back = tm.graph.connected_components(m.edges, min_len=1, nodes=None, engine=None)
# get mesh to uv mapping for the back vertices
m2u_body = mesh2UV(data_body.f.F, data_body.f.Ft)
m2u_body = filter_m2u(m2u_body, set(front))
m2u_body = convert_to_numbers(m2u_body)
m2u_front = mesh2UV(data_front.f.F, data_front.f.Ft)
m2u_front = convert_to_numbers(m2u_front)
# create mask for filtering vertices belonging to the back
body_front_mask = np.zeros((data_body.f.Vt.shape[0]), dtype=bool)
body_front_mask[front] = True
# define which points should match after the transform on the mesh
matching_points = np.array([ [9557, 12], [3284, 2], [3053, 5], [1888, 8], [10228, 4], [21, 7], [1209, 6], [1447, 3], [5908, 13], [4617, 15], [4219, 1], [6983, 10], [6989, 17], [7114, 9], [10602, 16], [10597, 11], [7884, 0], [8280, 14] ])
# matching_points = np.array([ [9557, 12], [3284, 2], [3053, 5], [1888, 8], [10228, 4], [21, 7], [1209, 6], [1447, 3], [5908, 13], [4793, 15], [4219, 1], [6983, 10], [6989, 17], [7114, 9], [10602, 16], [10597, 11], [7884, 0], [8450, 14], [13566, 18], [11767, 19], [8668, 20], [5012, 21], [12716, 22], [10917, 23], [8280, 24], [4617, 25], [2760, 26], [911, 27] ])
# map matching point to uv coordinates
for i in range(matching_points.shape[0]):
matching_points[i, :] = [get_masked_index(body_front_mask, m2u_body[matching_points[i,0]]), m2u_front[matching_points[i, 1]]]
x = data_body.f.Vt[body_front_mask, :]
y = uv_front
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(121)
# ax.set_xlim(0, 1)
# ax.set_ylim(0, 1)
ax.set_aspect('equal')
ax.scatter(x[:, 0], x[:, 1])
if y is not None:
ax.scatter(y[:, 0], y[:, 1])
for pair in matching_points:
ax.scatter([x[pair[0], 0], y[pair[1], 0]], [x[pair[0], 1], y[pair[1], 1]], s=50)
# get transformation from body to front contour
source = x[matching_points[:, 0], :]
destination = y[matching_points[:, 1], :]
trans = tps.TPS(source, destination)
transformed = trans(data_body.f.Vt[front, :])
# transformed = (transformed - transformed.min()) / (transformed.max() - transformed.min())
ax = fig.add_subplot(122)
# ax.set_xlim(0, 1)
# ax.set_ylim(0, 1)
ax.set_aspect('equal')
ax.scatter(transformed[:, 0], transformed[:, 1])
if y is not None:
ax.scatter(y[:, 0], y[:, 1])
for pair in matching_points:
ax.scatter([transformed[pair[0], 0], y[pair[1], 0]], [transformed[pair[0], 1], y[pair[1], 1]], s=50)
plt.show()
data_front_high = np.load("custom_data/uv_map/uv_data_plane_stretched_front_high.npz", allow_pickle=True)
y_high = data_front_high.f.Vt
# y_high[:, 0] = y_high[:, 0] + 0.1
# y_high[:, 1] = y_high[:, 1] + 0.42
closest_points = get_closest_points_normals(y_high, transformed)
# closest_points = get_closest_point(y_high, transformed)
unique_mask = np.zeros_like(closest_points, dtype=bool)
unique_mask[np.unique(closest_points, return_index=True)[1]] = True
matching_points = np.array([closest_points[unique_mask], np.array(range(y_high.shape[0]))[unique_mask]]).T
# matching_points_2 = []
# for i, p in enumerate(matching_points):
# if i % 10 == 0:
# matching_points_2.append(p)
# matching_points = np.array(matching_points_2)
source_2 = transformed[matching_points[:, 0], :]
destination_2 = y_high[matching_points[:, 1], :]
trans_2 = tps.TPS(source_2, destination_2)
transformed_2 = trans_2(transformed)
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(121)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.set_aspect('equal')
ax.scatter(source_2[:, 0], source_2[:, 1])
ax.scatter(destination_2[:, 0], destination_2[:, 1])
for i, pair in enumerate(zip(source_2, destination_2)):
if i % 5 == 0:
s, d = pair
ax.scatter([s[0], d[0]], [s[1], d[1]])
ax = fig.add_subplot(122)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.set_aspect('equal')
ax.scatter(transformed_2[:, 0], transformed_2[:, 1])
ax.scatter(y_high[:, 0], y_high[:, 1])
plt.show()
return transformed_2, mesh2UV(data_body.f.F, data_body.f.Ft), front, back
def smoothing():
data_body = np.load("Blender/uv_data_smpl_2.npz", allow_pickle=True)
uv_map = np.load('MeshUtils\stretched_uv_map_3.npy')
verts = np.concatenate((uv_map, np.ones((15068, 1))), axis=-1)
m = tm.Trimesh(verts, data_body.f.Ft, process=False, validate=False)
m2u_body_all = mesh2UV(data_body.f.F, data_body.f.Ft)
m2u_body_all = m2uv_array(m2u_body_all)
border_pairs = m2u_body_all[:, 0] != m2u_body_all[:, 1]
border_pairs = m2u_body_all[border_pairs, :]
pinned_vertices = border_pairs.reshape(-1)
lap = tm.smoothing.laplacian_calculation(m, equal_weight=False, pinned_vertices=pinned_vertices)
# tm.smoothing.filter_laplacian(m, lamb=0.5, iterations=100, volume_constraint=False, laplacian_operator=lap)
# tm.smoothing.filter_mut_dif_laplacian(m, lamb=0.5, iterations=100, volume_constraint=False, laplacian_operator=lap)
# tm.smoothing.filter_humphrey(m, iterations=100, laplacian_operator=lap)
tm.smoothing.filter_taubin(m, iterations=100, laplacian_operator=lap)
x = uv_map
y = m.vertices[:, :2]
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(211)
ax.set_xlim(0, 2)
ax.set_ylim(0, 1)
# ax.set_aspect('equal')
ax.scatter(x[:, 0], x[:, 1])
ax = fig.add_subplot(212)
ax.set_xlim(0, 2)
ax.set_ylim(0, 1)
# ax.set_aspect('equal')
ax.scatter(y[:, 0], y[:, 1])
plt.show()
pass
if __name__ == '__main__':
# generate_mask_2()
# smoothing()
# generate_uv_mask_border()
front_uv, m2uv, front_mask, back_mask = front()
back_uv = match_back_to_front(front_uv)
# back_uv = back()
final = np.zeros((15068, 2))
final[front_mask] = front_uv
final[back_mask] = back_uv + [1, 0]
np.save('MeshUtils\stretched_uv_map_3.npy', final)
pass
custom_utils/create_uv_map_2.py
+846
View File
@@ -0,0 +1,846 @@
import os
import sys
import shutil
import tqdm
import random
import math
import collections
from tps_custom import TPS
import numpy as np
import scipy as sp
import trimesh as tm
import cmapy as cm
import scipy.io as sio
import matplotlib.pyplot as plt
import plotly.express as ex
import plotly.graph_objects as go
from scipy.spatial.distance import cdist
def m2uv_array(m2uv, is_smpl_orig=False):
result = np.zeros((14475 if not is_smpl_orig else 3638, 2), dtype=np.int32)
for k, v in m2uv.items():
if len(v) == 2:
result[k] = list(v)
elif len(v) == 1:
x = v.pop()
result[k] = [x, x]
else:
assert False
return result
def get_body():
smpl = SMPL('FeatureLines/superSMPL')
pose = np.zeros((1, 72), dtype=np.float32)
beta = np.zeros((1, 11), dtype=np.float32)
R = smpl.run_rodrigues(pose)
T, _, _ = smpl.run(R, beta, True)
return np.squeeze(T)
def plot_3d(x, y=None):
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(111, projection='3d')
ax.set_xlim3d(-1, 1)
ax.set_ylim3d(-1, 1)
ax.set_zlim3d(-1, 1)
ax.set_xlabel("X")
ax.set_ylabel("Y")
ax.set_zlabel("Z")
ax.scatter(x[:, 0], x[:, 1], x[:, 2], s=5)
if y is not None:
ax.scatter(y[:, 0], y[:, 1], y[:, 2], s=5)
plt.show()
def plot_uv(x, y=None):
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(111)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.scatter(x[:, 0], x[:, 1])
if y is not None:
ax.scatter(y[:, 0], y[:, 1])
plt.show()
"""
Mesh to UV map
Computes correspondences between 3D mesh and UV map
NOTE: 3D mesh vertices can have multiple correspondences with UV vertices
"""
def mesh2UV(F, Ft):
m2uv = {v: set() for f in F for v in f}
for f, ft in zip(F, Ft):
for v, vt in zip(f, ft):
m2uv[v].add(vt)
# m2uv = {k:list(v) for k,v in m2uv.items()}
return m2uv
def filter_m2u(m2u, to_keep):
for key in m2u:
if len(m2u[key]) == 2:
v = list(m2u[key])
if v[0] in to_keep:
m2u[key].remove(v[1])
else:
m2u[key].remove(v[0])
return m2u
def convert_to_numbers(m2u):
for key in m2u:
m2u[key] = m2u[key].pop()
return m2u
def get_masked_index(mask, index):
m = np.zeros((mask.shape[0]), dtype=bool)
m[index] = True
m = m[mask]
new_index = np.argmax(m)
return new_index
def get_closest_point(verts_1, verts_2):
dist = sp.spatial.distance.cdist(verts_1, verts_2)
closest = np.argmin(dist, axis=-1)
return closest
def get_normal_edge(p1, p2):
d = p2 - p1
return np.array([-d[1], d[0]])
def get_normal_vertex(p1, p2, p3):
n1 = get_normal_edge(p1, p2)
n2 = get_normal_edge(p2, p3)
n = (n1 + n2)
n = n / np.linalg.norm(n) / 20
return n
def get_distance(p1, p2, points):
d = np.array([np.linalg.norm(np.cross(p2 - p1, point - p1)) / np.linalg.norm(p2 - p1) for point in points])
return d
def get_normals(verts):
v = verts.copy()
v = np.concatenate([v[-1][np.newaxis, :], v, v[0][np.newaxis, :]])
normals = np.zeros_like(verts)
for i in range(1, v.shape[0] - 1):
normals[i - 1] = get_normal_vertex(v[i - 1], v[i], v[i + 1])
return normals
def get_closest_points_normals(verts1, verts2):
dist_2 = sp.spatial.distance.cdist(verts1, verts2)
normals = get_normals(verts1)
# fig = plt.figure(figsize=(15, 15))
# ax = fig.add_subplot(111)
# ax.set_xlim(0, 1)
# ax.set_ylim(0, 1)
# ax.set_aspect('equal')
# ax.scatter(verts1[:, 0], verts1[:, 1])
# for p1, p2 in zip(verts1, verts1 + normals):
# ax.plot([p1[0], p2[0]], [p1[1], p2[1]])
# plt.show()
closest = []
i = 0
for p1, p2 in zip(verts1, verts1 + normals):
dist = get_distance(p1, p2, verts2)
closest.append(np.argmin(dist + dist_2[i, :]))
i += 1
return np.array(closest)
def back():
# load data
data_body = np.load("Blender/uv_data_1.npz", allow_pickle=True)
data_back = np.load("Blender/uv_data_plane_1_2.npz", allow_pickle=True)
# translate back uv to match body
uv_back = data_back.f.Vt
# uv_back[:, 0] = uv_back[:, 0] + 0.1
# uv_back[:, 1] = uv_back[:, 1] + 0.42
# find front and back indices in uv
uv_verts = np.append(data_body.f.Vt, np.zeros((data_body.f.Vt.shape[0], 1)), axis=-1)
m = tm.Trimesh(uv_verts, data_body.f.Ft, process=False, validate=False)
front, back = tm.graph.connected_components(m.edges, min_len=1, nodes=None, engine=None)
# get mesh to uv mapping for the back vertices
m2u_body = mesh2UV(data_body.f.F, data_body.f.Ft)
m2u_body = filter_m2u(m2u_body, set(back))
m2u_body = convert_to_numbers(m2u_body)
m2u_back = mesh2UV(data_back.f.F, data_back.f.Ft)
m2u_back = convert_to_numbers(m2u_back)
# create mask for filtering vertices belonging to the back
body_back_mask = np.zeros((data_body.f.Vt.shape[0]), dtype=bool)
body_back_mask[back] = True
# define which points should match after the transform on the mesh
matching_points = np.array([[3284, 3], [9557, 13], [9278, 15], [8095, 1], [10597, 10], [10602, 17], [7114, 9], [6989, 16], [6983, 11], [4219, 0], [5628, 14], [5908, 12], [1447, 2], [1086, 5], [21, 8], [33, 4], [1888, 7], [2931, 6]])
# map matching point to uv coordinates
for i in range(matching_points.shape[0]):
matching_points[i, :] = [get_masked_index(body_back_mask, m2u_body[matching_points[i,0]]), m2u_back[matching_points[i, 1]]]
x = data_body.f.Vt[body_back_mask, :]
y = uv_back
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(121)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.set_aspect('equal')
ax.scatter(x[:, 0], x[:, 1])
if y is not None:
ax.scatter(y[:, 0], y[:, 1])
for pair in matching_points:
ax.scatter([x[pair[0], 0], y[pair[1], 0]], [x[pair[0], 1], y[pair[1], 1]], s=50)
# get transformation from body to back contour
source = x[matching_points[:, 0], :]
destination = y[matching_points[:, 1], :]
trans = tps.TPS(source, destination)
transformed = trans(data_body.f.Vt[back, :])
ax = fig.add_subplot(122)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.set_aspect('equal')
ax.scatter(transformed[:, 0], transformed[:, 1])
if y is not None:
ax.scatter(y[:, 0], y[:, 1])
for pair in matching_points:
ax.scatter([transformed[pair[0], 0], y[pair[1], 0]], [transformed[pair[0], 1], y[pair[1], 1]], s=50)
plt.show()
data_back_high = np.load("Blender/uv_data_plane_1_high_res.npz", allow_pickle=True)
y_high = data_back_high.f.Vt
# y_high[:, 0] = y_high[:, 0] + 0.1
# y_high[:, 1] = y_high[:, 1] + 0.42
closest_points = get_closest_points_normals(y_high, transformed)
# closest_points = get_closest_point(y_high, transformed)
unique_mask = np.zeros_like(closest_points, dtype=bool)
unique_mask[np.unique(closest_points, return_index=True)[1]] = True
matching_points = np.array([closest_points[unique_mask], np.array(range(y_high.shape[0]))[unique_mask]]).T
# matching_points_2 = []
# for i, p in enumerate(matching_points):
# if i % 10 == 0:
# matching_points_2.append(p)
# matching_points = np.array(matching_points_2)
source_2 = transformed[matching_points[:, 0], :]
destination_2 = y_high[matching_points[:, 1], :]
trans_2 = tps.TPS(source_2, destination_2)
transformed_2 = trans_2(transformed)
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(121)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.set_aspect('equal')
ax.scatter(source_2[:, 0], source_2[:, 1])
ax.scatter(destination_2[:, 0], destination_2[:, 1])
for i, pair in enumerate(zip(source_2, destination_2)):
if i % 5 == 0:
s, d = pair
ax.scatter([s[0], d[0]], [s[1], d[1]])
ax = fig.add_subplot(122)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.set_aspect('equal')
ax.scatter(transformed_2[:, 0], transformed_2[:, 1])
ax.scatter(y_high[:, 0], y_high[:, 1])
plt.show()
return transformed_2
def match_back_to_front(front_uv):
# load data
data_body = np.load("Blender/uv_data_smpl_2.npz", allow_pickle=True)
# data_back = np.load("Blender/uv_data_plane_1_2.npz", allow_pickle=True)
# translate back uv to match body
# uv_back = data_back.f.Vt
# uv_back[:, 0] = uv_back[:, 0] + 0.1
# uv_back[:, 1] = uv_back[:, 1] + 0.42
# find front and back indices in uv
uv_verts = np.append(data_body.f.Vt, np.zeros((data_body.f.Vt.shape[0], 1)), axis=-1)
m = tm.Trimesh(uv_verts, data_body.f.Ft, process=False, validate=False)
front, back = tm.graph.connected_components(m.edges, min_len=1, nodes=None, engine=None)
# get mesh to uv mapping for the back vertices
m2u_body = mesh2UV(data_body.f.F, data_body.f.Ft)
m2u_body = filter_m2u(m2u_body, set(back))
m2u_body = convert_to_numbers(m2u_body)
# m2u_back = mesh2UV(data_back.f.F, data_back.f.Ft)
# m2u_back = convert_to_numbers(m2u_back)
# create mask for filtering vertices belonging to the back
body_back_mask = np.zeros((data_body.f.Vt.shape[0]), dtype=bool)
body_back_mask[back] = True
body_front_mask = np.zeros((data_body.f.Vt.shape[0]), dtype=bool)
body_front_mask[front] = True
# define which points should match after the transform on the mesh
# matching_points = np.array([[3284, 3], [9557, 13], [9278, 15], [8095, 1], [10597, 10], [10602, 17], [7114, 9], [6989, 16], [6983, 11], [4219, 0], [5628, 14], [5908, 12], [1447, 2], [1086, 5], [21, 8], [33, 4], [1888, 7], [2931, 6]])
# # map matching point to uv coordinates
# for i in range(matching_points.shape[0]):
# matching_points[i, :] = [get_masked_index(body_back_mask, m2u_body[matching_points[i,0]]), m2u_back[matching_points[i, 1]]]
m2u_body_all = mesh2UV(data_body.f.F, data_body.f.Ft)
m2u_body_all = m2uv_array(m2u_body_all)
border_pairs = m2u_body_all[:, 0] != m2u_body_all[:, 1]
border_pairs = m2u_body_all[border_pairs, :]
matching_points = border_pairs
matching_points_sorted = []
for p in matching_points:
if p[0] in front:
matching_points_sorted.append([p[0], p[1]])
elif p[0] in back:
matching_points_sorted.append([p[1], p[0]])
else:
assert False
# add neck V shape control point
matching_points_sorted.append([m2u_body_all[10228][0], m2u_body_all[33][0]])
matching_points = np.array(matching_points_sorted)
mm = matching_points.copy()
for i in range(matching_points.shape[0]):
matching_points[i, :] = [get_masked_index(body_back_mask, matching_points[i,1]), get_masked_index(body_front_mask, matching_points[i,0])]
temp = np.zeros_like(data_body.f.Vt)
temp[front, :] = front_uv
x = data_body.f.Vt[body_back_mask, :]
y = temp[body_front_mask, :] # front_uv #
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(121)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.set_aspect('equal')
ax.scatter(x[:, 0], x[:, 1])
if y is not None:
ax.scatter(y[:, 0], y[:, 1])
for pair in matching_points:
ax.scatter([x[pair[0], 0], y[pair[1], 0]], [x[pair[0], 1], y[pair[1], 1]], s=50)
# get transformation from body to back contour
source = x[matching_points[:, 0], :]
destination = y[matching_points[:, 1], :]
trans = tps.TPS(source, destination)
transformed = trans(data_body.f.Vt[back, :])
ax = fig.add_subplot(122)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.set_aspect('equal')
ax.scatter(transformed[:, 0], transformed[:, 1])
if y is not None:
ax.scatter(y[:, 0], y[:, 1])
for pair in matching_points:
ax.scatter([transformed[pair[0], 0], y[pair[1], 0]], [transformed[pair[0], 1], y[pair[1], 1]], s=50)
plt.show()
return transformed
def generate_mask():
import cv2
res = 1024
front = np.load("Blender/uv_data_plane_0_high_res_2.npz", allow_pickle=True)
back = np.load("Blender/uv_data_plane_1_high_res_2.npz", allow_pickle=True)
points = np.round(np.concatenate((front.f.Vt, back.f.Vt)) * res).astype(np.int32)
img = np.zeros((res, res, 3))
img[points[:, 0], points[:, 1], :] = 255
cv2.imwrite('MeshUtils/mask_1024.png', img)
# do paint now
img = cv2.imread('MeshUtils/mask_1024_2.png')
img = img[:, :, 0] == 255
np.save('MeshUtils/mask_1024.npy', img)
def generate_mask_2():
import cv2
res = 256
front = np.load("Blender/uv_data_plane_stretched_front_high_extra.npz", allow_pickle=True)
back = np.load("Blender/uv_data_plane_stretched_front_high_extra.npz", allow_pickle=True)
points = np.floor(np.concatenate((front.f.Vt, back.f.Vt + [1, 0])) * res).astype(np.int32)
img = np.zeros((res * 2, res, 3))
img[points[:, 0], points[:, 1], :] = 255
cv2.imwrite('MeshUtils/mask_256x512.png', img)
# do paint now
img = cv2.imread('MeshUtils/mask_256x512.png')
img = img[:, :, 0] > 240
np.save('MeshUtils/mask_256x512.npy', img)
pass
def generate_uv_mask_border():
import cv2
mask = np.load('MeshUtils/mask_256x512.npy')
uv_res = 256
mask_border = (cv2.resize(mask * 255.0, (uv_res, uv_res * 2)) / 255).astype(bool)[:, :, np.newaxis]
mask_border = cv2.erode(mask_border.astype(np.float32), cv2.getStructuringElement(cv2.MORPH_RECT, (2 * 2 + 1, 2 * 2 + 1), (2, 2)), iterations=1).astype(bool)
mask[mask_border == True] = 0
# plt.imshow(mask)
idx_front = np.transpose(np.vstack(np.nonzero(mask[:256, :])), [1,0])
test = np.zeros_like(mask)
test[idx_front[:,0], idx_front[:,1]] = 1
idx_back = idx_front.copy()
idx_back[:, 0] += 256
test[idx_back[:,0], idx_back[:,1]] = 1
pass
def front():
# load data
data_body = np.load("custom_data/uv_map/uv_data_smpl_2.npz", allow_pickle=True)
data_front = np.load("custom_data/uv_map/uv_data_plane_stretched_front_low.npz", allow_pickle=True)
# translate back uv to match body
uv_front = data_front.f.Vt
# uv_back[:, 0] = uv_back[:, 0] + 0.1
# uv_back[:, 1] = uv_back[:, 1] + 0.42
# find front and back indices in uv
uv_verts = np.append(data_body.f.Vt, np.zeros((data_body.f.Vt.shape[0], 1)), axis=-1)
m = tm.Trimesh(uv_verts, data_body.f.Ft, process=False, validate=False)
front, back = tm.graph.connected_components(m.edges, min_len=1, nodes=None, engine=None)
# get mesh to uv mapping for the back vertices
m2u_body = mesh2UV(data_body.f.F, data_body.f.Ft)
m2u_body = filter_m2u(m2u_body, set(front))
m2u_body = convert_to_numbers(m2u_body)
m2u_front = mesh2UV(data_front.f.F, data_front.f.Ft)
m2u_front = convert_to_numbers(m2u_front)
# create mask for filtering vertices belonging to the back
body_front_mask = np.zeros((data_body.f.Vt.shape[0]), dtype=bool)
body_front_mask[front] = True
# define which points should match after the transform on the mesh
matching_points = np.array([ [9557, 12], [3284, 2], [3053, 5], [1888, 8], [10228, 4], [21, 7], [1209, 6], [1447, 3], [5908, 13], [4617, 15], [4219, 1], [6983, 10], [6989, 17], [7114, 9], [10602, 16], [10597, 11], [7884, 0], [8280, 14] ])
# matching_points = np.array([ [9557, 12], [3284, 2], [3053, 5], [1888, 8], [10228, 4], [21, 7], [1209, 6], [1447, 3], [5908, 13], [4793, 15], [4219, 1], [6983, 10], [6989, 17], [7114, 9], [10602, 16], [10597, 11], [7884, 0], [8450, 14], [13566, 18], [11767, 19], [8668, 20], [5012, 21], [12716, 22], [10917, 23], [8280, 24], [4617, 25], [2760, 26], [911, 27] ])
# map matching point to uv coordinates
for i in range(matching_points.shape[0]):
matching_points[i, :] = [get_masked_index(body_front_mask, m2u_body[matching_points[i,0]]), m2u_front[matching_points[i, 1]]]
x = data_body.f.Vt[body_front_mask, :]
y = uv_front
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(121)
# ax.set_xlim(0, 1)
# ax.set_ylim(0, 1)
ax.set_aspect('equal')
ax.scatter(x[:, 0], x[:, 1])
if y is not None:
ax.scatter(y[:, 0], y[:, 1])
for pair in matching_points:
ax.scatter([x[pair[0], 0], y[pair[1], 0]], [x[pair[0], 1], y[pair[1], 1]], s=50)
# get transformation from body to front contour
source = x[matching_points[:, 0], :]
destination = y[matching_points[:, 1], :]
trans = tps.TPS(source, destination)
transformed = trans(data_body.f.Vt[front, :])
# transformed = (transformed - transformed.min()) / (transformed.max() - transformed.min())
ax = fig.add_subplot(122)
# ax.set_xlim(0, 1)
# ax.set_ylim(0, 1)
ax.set_aspect('equal')
ax.scatter(transformed[:, 0], transformed[:, 1])
if y is not None:
ax.scatter(y[:, 0], y[:, 1])
for pair in matching_points:
ax.scatter([transformed[pair[0], 0], y[pair[1], 0]], [transformed[pair[0], 1], y[pair[1], 1]], s=50)
plt.show()
data_front_high = np.load("custom_data/uv_map/uv_data_plane_stretched_front_high.npz", allow_pickle=True)
y_high = data_front_high.f.Vt
# y_high[:, 0] = y_high[:, 0] + 0.1
# y_high[:, 1] = y_high[:, 1] + 0.42
closest_points = get_closest_points_normals(y_high, transformed)
# closest_points = get_closest_point(y_high, transformed)
unique_mask = np.zeros_like(closest_points, dtype=bool)
unique_mask[np.unique(closest_points, return_index=True)[1]] = True
matching_points = np.array([closest_points[unique_mask], np.array(range(y_high.shape[0]))[unique_mask]]).T
# matching_points_2 = []
# for i, p in enumerate(matching_points):
# if i % 10 == 0:
# matching_points_2.append(p)
# matching_points = np.array(matching_points_2)
source_2 = transformed[matching_points[:, 0], :]
destination_2 = y_high[matching_points[:, 1], :]
trans_2 = tps.TPS(source_2, destination_2)
transformed_2 = trans_2(transformed)
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(121)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.set_aspect('equal')
ax.scatter(source_2[:, 0], source_2[:, 1])
ax.scatter(destination_2[:, 0], destination_2[:, 1])
for i, pair in enumerate(zip(source_2, destination_2)):
if i % 5 == 0:
s, d = pair
ax.scatter([s[0], d[0]], [s[1], d[1]])
ax = fig.add_subplot(122)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.set_aspect('equal')
ax.scatter(transformed_2[:, 0], transformed_2[:, 1])
ax.scatter(y_high[:, 0], y_high[:, 1])
plt.show()
return transformed_2, mesh2UV(data_body.f.F, data_body.f.Ft), front, back
def faces2graph(F, v=None):
if v: v = set(v) # Only this verts
G = {}
for f in F:
for i in f:
if v and i not in v: continue
if not i in G:
G[i] = {}
for j in f:
if v and j not in v: continue
if i == j: continue
G[i][j] = 1
return G
def custom_smoothing(mesh, edges, pinned, F, iterations=10):
# custom_scatter(mesh[pinned, :]).show()
mesh_orig = mesh.copy()
G = faces2graph(F)
for _ in tqdm.tqdm(range(iterations)):
edge_lengths = np.sqrt(((mesh[edges[:,0], :] - mesh[edges[:,1], :]) ** 2 ).sum(-1))
edge_lengths_m = np.zeros((15068, 15068), np.float32)
edge_lengths_m[edges[:,0], edges[:,1]] = edge_lengths
edge_lengths_m[edges[:,1], edges[:,0]] = edge_lengths
# mean_edge = edge_lengths.mean()
# dist = cdist(mesh, mesh)
# for i in range(dist.shape[0]):
# dist[i,i] = 10000
# dist = sp.spatial.KDTree(mesh)
# closest_neighbour = edge_lengths_m.argmin(axis=0)
# if edge_lengths_m[i, closest_neighbour[i]] > mean_edge:
# continue
mesh_temp = mesh.copy()
for i in range(15068):
if i in pinned:
continue
# closest_neighbour = edge_lengths_m[i, :].argmin()
# if edge_lengths_m[i, closest_neighbour] > mean_edge:
# continue
neighbours = list(G[i].keys())
neighbours_dist = edge_lengths_m[i, neighbours]
weights = neighbours_dist / neighbours_dist.sum()
weights = np.ones_like(weights) / weights.shape[0]
mesh_temp[i, :] = np.sum(mesh[neighbours, :] * weights[:, np.newaxis], axis=0)
# neighbours_dist, neighbours = dist.query(mesh[i,:], k=10, workers=10)
# # if neighbours[0] == 15068:
# # continue
# neighbours_dist, neighbours = neighbours_dist[1:], neighbours[1:]
# weights = neighbours_dist / neighbours_dist.sum()
# mesh_temp[i, :] = np.sum(mesh[neighbours, :] * weights[:, np.newaxis], axis=0)
# mesh_temp[i, :] = np.mean(mesh[neighbours, :], axis=0)
# print()
# closest_neighbour = edge_lengths_m[i, :].argmin()
# if edge_lengths_m[i, closest_neighbour] < mean_edge:
# vector = mesh[i,:] - mesh[closest_neighbour,:]
# vector = vector / np.sqrt(np.sum(vector[0] ** 2 + vector[1] ** 2 ))
# to_move = vector / 1000
# # to_move = vector / edge_lengths_m[i, closest_neighbour] * 100
# mesh[i, :] -= to_move
mesh = mesh_temp
custom_scatter(mesh_orig, mesh).show()
print()
return mesh
def custom_scatter(x, y=None):
data = [go.Scatter(
x=x[:,0],
y=x[:,1],
mode='markers',
marker=dict(
size=3,
line=dict(width=0)
),
),]
if y is not None:
data.append(
go.Scatter(
x=y[:,0],
y=y[:,1],
mode='markers',
marker=dict(
size=3,
line=dict(width=0)
),
),)
fig = go.Figure(data=data)
# fig.update_layout(scene_aspectmode='manual',
# scene_aspectratio=dict(x=1, y=1, z=1))
# fig.update_layout(
# scene = dict(
# xaxis = dict(nticks=4, range=[-1,1],),
# yaxis = dict(nticks=4, range=[-1,1],),
# zaxis = dict(nticks=4, range=[-1,1],),)
# )
return fig
def smoothing():
data_body = np.load("custom_data/uv_map/uv_data_smpl_2.npz", allow_pickle=True)
uv_map = np.load('custom_data/uv_map/stretched_uv_map_3.npy')
F = np.load("custom_utils/smpl_faces.npy").astype(np.int32)
verts = np.concatenate((uv_map, np.ones((15068, 1))), axis=-1)
m = tm.Trimesh(verts, data_body.f.Ft, process=False, validate=False)
m2u_body_all = mesh2UV(data_body.f.F, data_body.f.Ft)
m2u_body_all = m2uv_array(m2u_body_all)
border_pairs = m2u_body_all[:, 0] != m2u_body_all[:, 1]
border_pairs = m2u_body_all[border_pairs, :]
pinned_vertices = border_pairs.reshape(-1)
right_legs_bottom = np.nonzero(((verts[:,0] > 1).astype(np.int8) * (verts[:,0] < 1.03).astype(np.int8)).astype(bool))[0]
left_legs_bottom = np.nonzero(verts[:,0] < 0.025)[0]
top_arms = np.nonzero(verts[:,1] > 0.995)[0]
bottom_arms = np.nonzero(verts[:,1] < 0.005)[0]
pinned_vertices = np.concatenate((pinned_vertices, right_legs_bottom, left_legs_bottom, top_arms, bottom_arms))
x = custom_smoothing(np.asarray(m.vertices)[:,:2], np.asarray(m.edges), pinned_vertices, data_body.f.Ft)
# m = tm.Trimesh(x, data_body.f.Ft, process=False, validate=False)
lap = tm.smoothing.laplacian_calculation(m, equal_weight=False, pinned_vertices=pinned_vertices)
# tm.smoothing.filter_laplacian(m, lamb=0.5, implicit_time_integration=False, iterations=1000, volume_constraint=False, laplacian_operator=lap)
# tm.smoothing.filter_mut_dif_laplacian(m, lamb=0.9, iterations=100, volume_constraint=True, laplacian_operator=lap)
tm.smoothing.filter_humphrey(m, iterations=1000, laplacian_operator=lap, alpha=0.01, beta=1.0)
# tm.smoothing.filter_taubin(m, iterations=25, lamb=0.01, nu=0.0015, laplacian_operator=lap)
x = uv_map
y = m.vertices[:, :2]
fig = go.Figure(data=[
# go.Scatter(
# x=x[:,0],
# y=x[:,1],
# mode='markers',
# marker=dict(
# size=3,
# line=dict(width=0)
# ),
# ),
go.Scatter(
x=y[:,0],
y=y[:,1],
mode='markers',
marker=dict(
size=3,
line=dict(width=0)
),
),
])
fig.update_layout(scene_aspectmode='manual',
scene_aspectratio=dict(x=1, y=1, z=1))
fig.update_layout(
scene = dict(
xaxis = dict(nticks=4, range=[-1,1],),
yaxis = dict(nticks=4, range=[-1,1],),
zaxis = dict(nticks=4, range=[-1,1],),)
)
fig.show()
dist = cdist(y, y)
for i in range(dist.shape[0]):
dist[i,i] = 10000
res = 1 / dist.min()
print()
pass
def front_3dto2d():
# load data
data_body = np.load("custom_data/uv_map/uv_data_smpl_2.npz", allow_pickle=True)
data_front = np.load("custom_data/uv_map/uv_data_plane_stretched_front_low.npz", allow_pickle=True)
laplacian_loss = sio.loadmat("custom_utils/ssmpl0.mat")['L'].tocoo()
edges = np.stack(np.nonzero(laplacian_loss.todense())).transpose()
# translate back uv to match body
uv_front = data_front.f.V
# uv_back[:, 0] = uv_back[:, 0] + 0.1
# uv_back[:, 1] = uv_back[:, 1] + 0.42
# find front and back indices in uv
uv_verts = np.append(data_body.f.Vt, np.zeros((data_body.f.Vt.shape[0], 1)), axis=-1)
m = tm.Trimesh(uv_verts, data_body.f.Ft, process=False, validate=False)
front, back = tm.graph.connected_components(m.edges, min_len=1, nodes=None, engine=None)
# get mesh to uv mapping for the back vertices
m2u_body = mesh2UV(data_body.f.F, data_body.f.Ft)
m2u_body = filter_m2u(m2u_body, set(front))
m2u_body = convert_to_numbers(m2u_body)
m2u_front = mesh2UV(data_front.f.F, data_front.f.Ft)
m2u_front = convert_to_numbers(m2u_front)
# create mask for filtering vertices belonging to the back
body_front_mask = np.zeros((data_body.f.Vt.shape[0]), dtype=bool)
body_front_mask[front] = True
# define which points should match after the transform on the mesh
matching_points = np.array([ [9557, 12], [3284, 2], [3053, 5], [1888, 8], [10228, 4], [21, 7], [1209, 6], [1447, 3], [5908, 13], [4617, 15], [4219, 1], [6983, 10], [6989, 17], [7114, 9], [10602, 16], [10597, 11], [7884, 0], [8280, 14] ])
# for i in range(matching_points.shape[0]):
# matching_points[i, :] = [get_masked_index(body_front_mask, matching_points[i,0]),matching_points[i, 1]]
x = data_body.f.V
# x = data_body.f.V[body_front_mask, :]
y = uv_front
# get transformation from body to front contour
source = x[matching_points[:, 0], :]
destination = y[matching_points[:, 1], :]
trans = TPS(source, destination)
transformed = trans(data_body.f.V)
z = transformed
fig = go.Figure(data=[
# go.Scatter3d(
# x=x[:,0],
# y=x[:,1],
# z=x[:,2],
# ),
go.Scatter3d(
x=y[:,0],
y=y[:,1],
z=y[:,2],
mode='markers',
marker=dict(
size=10,
line=dict(width=0)
),
),
go.Scatter3d(
x=z[:,0],
y=z[:,1],
z=z[:,2],
mode='markers',
marker=dict(
size=10,
line=dict(width=0)
),
),
])
fig.update_layout(scene_aspectmode='manual',
scene_aspectratio=dict(x=1, y=1, z=1))
fig.update_layout(
scene = dict(
xaxis = dict(nticks=4, range=[-1,1],),
yaxis = dict(nticks=4, range=[-1,1],),
zaxis = dict(nticks=4, range=[-1,1],),)
)
fig.show()
print()
if __name__ == '__main__':
# generate_mask_2()
smoothing()
# generate_uv_mask_border()
# front_uv, m2uv, front_mask, back_mask = front_3dto2d()
# back_uv = match_back_to_front(front_uv)
# # back_uv = back()
# final = np.zeros((15068, 2))
# final[front_mask] = front_uv
# final[back_mask] = back_uv + [1, 0]
# np.save('MeshUtils\stretched_uv_map_3.npy', final)
pass
custom_utils/tps_custom.py
+210
View File
@@ -0,0 +1,210 @@
import numpy
import pygeodesic.geodesic as geodesic
__all__ = ['find_coefficients', 'transform']
def cdist(K: numpy.ndarray, B: numpy.ndarray) -> numpy.ndarray:
"""Calculate Euclidean distance between K[i, :] and B[j, :].
Arguments
---------
K : numpy.array
B : numpy.array
"""
K = numpy.atleast_2d(K)
B = numpy.atleast_2d(B)
assert K.ndim == 2
assert B.ndim == 2
K = numpy.expand_dims(K, 1)
B = numpy.expand_dims(B, 0)
D = K - B
return numpy.linalg.norm(D, axis=2)
# def cdist(K: numpy.ndarray, B: numpy.ndarray, faces: numpy.ndarray) -> numpy.ndarray:
# geoalg = geodesic.PyGeodesicAlgorithmExact(points, faces)
def pairwise_radial_basis(K: numpy.ndarray, B: numpy.ndarray) -> numpy.ndarray:
"""Compute the TPS radial basis function phi(r) between every row-pair of K
and B where r is the Euclidean distance.
Arguments
---------
K : numpy.array
n by d vector containing n d-dimensional points.
B : numpy.array
m by d vector containing m d-dimensional points.
Return
------
P : numpy.array
n by m matrix where.
P(i, j) = phi( norm( K(i,:) - B(j,:) ) ),
where phi(r) = r^2*log(r), if r >= 1
r*log(r^r), if r < 1
"""
# r_mat(i, j) is the Euclidean distance between K(i, :) and B(j, :).
r_mat = cdist(K, B)
pwise_cond_ind1 = r_mat >= 1
pwise_cond_ind2 = r_mat < 1
r_mat_p1 = r_mat[pwise_cond_ind1]
r_mat_p2 = r_mat[pwise_cond_ind2]
# P correcponds to the matrix K from [1].
P = numpy.empty(r_mat.shape)
P[pwise_cond_ind1] = (r_mat_p1**2) * numpy.log(r_mat_p1)
P[pwise_cond_ind2] = r_mat_p2 * numpy.log(numpy.power(r_mat_p2, r_mat_p2))
return P
def find_coefficients(control_points: numpy.ndarray,
target_points: numpy.ndarray,
lambda_: float = 0.,
solver: str = 'exact') -> numpy.ndarray:
"""Given a set of control points and their corresponding points, compute the
coefficients of the TPS interpolant deforming surface.
Arguments
---------
control_points : numpy.array
p by d vector of control points
target_points : numpy.array
p by d vector of corresponding target points on the deformed
surface
lambda_ : float
regularization parameter
solver : str
the solver to get the coefficients. default is 'exact' for the exact
solution. Or use 'lstsq' for the least square solution.
Return
------
coef : numpy.ndarray
the coefficients
.. seealso::
http://cseweb.ucsd.edu/~sjb/pami_tps.pdf
"""
# ensure data type and shape
control_points = numpy.atleast_2d(control_points)
target_points = numpy.atleast_2d(target_points)
if control_points.shape != target_points.shape:
raise ValueError(
'Shape of and control points {cp} and target points {tp} are not the same.'.
format(cp=control_points.shape, tp=target_points.shape))
p, d = control_points.shape
# The matrix
K = pairwise_radial_basis(control_points, control_points)
P = numpy.hstack([numpy.ones((p, 1)), control_points])
# Relax the exact interpolation requirement by means of regularization.
K = K + lambda_ * numpy.identity(p)
# Target points
M = numpy.vstack([
numpy.hstack([K, P]),
numpy.hstack([P.T, numpy.zeros((d + 1, d + 1))])
])
Y = numpy.vstack([target_points, numpy.zeros((d + 1, d))])
# solve for M*X = Y.
# At least d+1 control points should not be in a subspace; e.g. for d=2, at
# least 3 points are not on a straight line. Otherwise M will be singular.
solver = solver.lower()
if solver == 'exact':
X = numpy.linalg.solve(M, Y)
elif solver == 'lstsq':
X, _, _, _ = numpy.linalg.lstsq(M, Y, None)
else:
raise ValueError('Unknown solver: ' + solver)
return X
def transform(source_points: numpy.ndarray, control_points: numpy.ndarray,
coefficient: numpy.ndarray) -> numpy.ndarray:
"""Transform the source points form the original surface to the destination
(deformed) surface.
Arguments
---------
source_points : numpy.array
n by d array of source points to be transformed
control_points : numpy.array
the control points used in the function `find_coefficients`
coefficient : numpy.array
the computed coefficients
Return
------
deformed_points : numpy.array
n by d array of the transformed point on the target surface
"""
source_points = numpy.atleast_2d(source_points)
control_points = numpy.atleast_2d(control_points)
if source_points.shape[-1] != control_points.shape[-1]:
raise ValueError(
'Dimension of source points ({sd}D) and control points ({cd}D) are not the same.'.
format(sd=source_points.shape[-1], cd=control_points.shape[-1]))
n = source_points.shape[0]
A = pairwise_radial_basis(source_points, control_points)
K = numpy.hstack([A, numpy.ones((n, 1)), source_points])
deformed_points = numpy.dot(K, coefficient)
return deformed_points
class TPS:
"""The thin plate spline deformation warpping.
"""
def __init__(self,
control_points: numpy.ndarray,
target_points: numpy.ndarray,
lambda_: float = 0.,
solver: str = 'exact'):
"""Create a instance that preserve the TPS coefficients.
Arguments
---------
control_points : numpy.array
p by d vector of control points
target_points : numpy.array
p by d vector of corresponding target points on the deformed
surface
lambda_ : float
regularization parameter
solver : str
the solver to get the coefficients. default is 'exact' for the
exact solution. Or use 'lstsq' for the least square solution.
"""
self.control_points = control_points
self.coefficient = find_coefficients(
control_points, target_points, lambda_, solver)
def __call__(self, source_points):
"""Transform the source points form the original surface to the
destination (deformed) surface.
Arguments
---------
source_points : numpy.array
n by d array of source points to be transformed
"""
return transform(source_points, self.control_points,
self.coefficient)
transform = __call__