846 lines
28 KiB
Python
846 lines
28 KiB
Python
import os
|
|
import sys
|
|
import shutil
|
|
import tqdm
|
|
import random
|
|
import math
|
|
import collections
|
|
from tps_custom import TPS
|
|
import numpy as np
|
|
import scipy as sp
|
|
import trimesh as tm
|
|
import cmapy as cm
|
|
import scipy.io as sio
|
|
import matplotlib.pyplot as plt
|
|
import plotly.express as ex
|
|
import plotly.graph_objects as go
|
|
from scipy.spatial.distance import cdist
|
|
|
|
|
|
|
|
def m2uv_array(m2uv, is_smpl_orig=False):
|
|
result = np.zeros((14475 if not is_smpl_orig else 3638, 2), dtype=np.int32)
|
|
for k, v in m2uv.items():
|
|
if len(v) == 2:
|
|
result[k] = list(v)
|
|
elif len(v) == 1:
|
|
x = v.pop()
|
|
result[k] = [x, x]
|
|
else:
|
|
assert False
|
|
|
|
return result
|
|
|
|
|
|
def get_body():
|
|
smpl = SMPL('FeatureLines/superSMPL')
|
|
pose = np.zeros((1, 72), dtype=np.float32)
|
|
beta = np.zeros((1, 11), dtype=np.float32)
|
|
R = smpl.run_rodrigues(pose)
|
|
T, _, _ = smpl.run(R, beta, True)
|
|
return np.squeeze(T)
|
|
|
|
|
|
def plot_3d(x, y=None):
|
|
fig = plt.figure(figsize=(15, 15))
|
|
ax = fig.add_subplot(111, projection='3d')
|
|
ax.set_xlim3d(-1, 1)
|
|
ax.set_ylim3d(-1, 1)
|
|
ax.set_zlim3d(-1, 1)
|
|
ax.set_xlabel("X")
|
|
ax.set_ylabel("Y")
|
|
ax.set_zlabel("Z")
|
|
ax.scatter(x[:, 0], x[:, 1], x[:, 2], s=5)
|
|
if y is not None:
|
|
ax.scatter(y[:, 0], y[:, 1], y[:, 2], s=5)
|
|
plt.show()
|
|
|
|
def plot_uv(x, y=None):
|
|
fig = plt.figure(figsize=(15, 15))
|
|
ax = fig.add_subplot(111)
|
|
ax.set_xlim(0, 1)
|
|
ax.set_ylim(0, 1)
|
|
ax.scatter(x[:, 0], x[:, 1])
|
|
if y is not None:
|
|
ax.scatter(y[:, 0], y[:, 1])
|
|
plt.show()
|
|
|
|
"""
|
|
Mesh to UV map
|
|
Computes correspondences between 3D mesh and UV map
|
|
NOTE: 3D mesh vertices can have multiple correspondences with UV vertices
|
|
"""
|
|
def mesh2UV(F, Ft):
|
|
m2uv = {v: set() for f in F for v in f}
|
|
for f, ft in zip(F, Ft):
|
|
for v, vt in zip(f, ft):
|
|
m2uv[v].add(vt)
|
|
# m2uv = {k:list(v) for k,v in m2uv.items()}
|
|
return m2uv
|
|
|
|
|
|
def filter_m2u(m2u, to_keep):
|
|
for key in m2u:
|
|
if len(m2u[key]) == 2:
|
|
v = list(m2u[key])
|
|
if v[0] in to_keep:
|
|
m2u[key].remove(v[1])
|
|
else:
|
|
m2u[key].remove(v[0])
|
|
return m2u
|
|
|
|
|
|
def convert_to_numbers(m2u):
|
|
for key in m2u:
|
|
m2u[key] = m2u[key].pop()
|
|
return m2u
|
|
|
|
|
|
def get_masked_index(mask, index):
|
|
m = np.zeros((mask.shape[0]), dtype=bool)
|
|
m[index] = True
|
|
m = m[mask]
|
|
new_index = np.argmax(m)
|
|
return new_index
|
|
|
|
|
|
def get_closest_point(verts_1, verts_2):
|
|
dist = sp.spatial.distance.cdist(verts_1, verts_2)
|
|
closest = np.argmin(dist, axis=-1)
|
|
return closest
|
|
|
|
def get_normal_edge(p1, p2):
|
|
d = p2 - p1
|
|
return np.array([-d[1], d[0]])
|
|
|
|
def get_normal_vertex(p1, p2, p3):
|
|
n1 = get_normal_edge(p1, p2)
|
|
n2 = get_normal_edge(p2, p3)
|
|
n = (n1 + n2)
|
|
n = n / np.linalg.norm(n) / 20
|
|
return n
|
|
|
|
|
|
def get_distance(p1, p2, points):
|
|
d = np.array([np.linalg.norm(np.cross(p2 - p1, point - p1)) / np.linalg.norm(p2 - p1) for point in points])
|
|
return d
|
|
|
|
|
|
def get_normals(verts):
|
|
v = verts.copy()
|
|
v = np.concatenate([v[-1][np.newaxis, :], v, v[0][np.newaxis, :]])
|
|
normals = np.zeros_like(verts)
|
|
for i in range(1, v.shape[0] - 1):
|
|
normals[i - 1] = get_normal_vertex(v[i - 1], v[i], v[i + 1])
|
|
return normals
|
|
|
|
|
|
def get_closest_points_normals(verts1, verts2):
|
|
dist_2 = sp.spatial.distance.cdist(verts1, verts2)
|
|
normals = get_normals(verts1)
|
|
|
|
# fig = plt.figure(figsize=(15, 15))
|
|
# ax = fig.add_subplot(111)
|
|
# ax.set_xlim(0, 1)
|
|
# ax.set_ylim(0, 1)
|
|
# ax.set_aspect('equal')
|
|
# ax.scatter(verts1[:, 0], verts1[:, 1])
|
|
# for p1, p2 in zip(verts1, verts1 + normals):
|
|
# ax.plot([p1[0], p2[0]], [p1[1], p2[1]])
|
|
# plt.show()
|
|
|
|
closest = []
|
|
i = 0
|
|
for p1, p2 in zip(verts1, verts1 + normals):
|
|
dist = get_distance(p1, p2, verts2)
|
|
closest.append(np.argmin(dist + dist_2[i, :]))
|
|
i += 1
|
|
|
|
return np.array(closest)
|
|
|
|
|
|
def back():
|
|
# load data
|
|
data_body = np.load("Blender/uv_data_1.npz", allow_pickle=True)
|
|
data_back = np.load("Blender/uv_data_plane_1_2.npz", allow_pickle=True)
|
|
|
|
# translate back uv to match body
|
|
uv_back = data_back.f.Vt
|
|
# uv_back[:, 0] = uv_back[:, 0] + 0.1
|
|
# uv_back[:, 1] = uv_back[:, 1] + 0.42
|
|
|
|
# find front and back indices in uv
|
|
uv_verts = np.append(data_body.f.Vt, np.zeros((data_body.f.Vt.shape[0], 1)), axis=-1)
|
|
m = tm.Trimesh(uv_verts, data_body.f.Ft, process=False, validate=False)
|
|
front, back = tm.graph.connected_components(m.edges, min_len=1, nodes=None, engine=None)
|
|
|
|
# get mesh to uv mapping for the back vertices
|
|
m2u_body = mesh2UV(data_body.f.F, data_body.f.Ft)
|
|
m2u_body = filter_m2u(m2u_body, set(back))
|
|
m2u_body = convert_to_numbers(m2u_body)
|
|
|
|
m2u_back = mesh2UV(data_back.f.F, data_back.f.Ft)
|
|
m2u_back = convert_to_numbers(m2u_back)
|
|
|
|
# create mask for filtering vertices belonging to the back
|
|
body_back_mask = np.zeros((data_body.f.Vt.shape[0]), dtype=bool)
|
|
body_back_mask[back] = True
|
|
|
|
# define which points should match after the transform on the mesh
|
|
matching_points = np.array([[3284, 3], [9557, 13], [9278, 15], [8095, 1], [10597, 10], [10602, 17], [7114, 9], [6989, 16], [6983, 11], [4219, 0], [5628, 14], [5908, 12], [1447, 2], [1086, 5], [21, 8], [33, 4], [1888, 7], [2931, 6]])
|
|
|
|
# map matching point to uv coordinates
|
|
for i in range(matching_points.shape[0]):
|
|
matching_points[i, :] = [get_masked_index(body_back_mask, m2u_body[matching_points[i,0]]), m2u_back[matching_points[i, 1]]]
|
|
|
|
x = data_body.f.Vt[body_back_mask, :]
|
|
y = uv_back
|
|
|
|
fig = plt.figure(figsize=(15, 15))
|
|
ax = fig.add_subplot(121)
|
|
ax.set_xlim(0, 1)
|
|
ax.set_ylim(0, 1)
|
|
ax.set_aspect('equal')
|
|
ax.scatter(x[:, 0], x[:, 1])
|
|
if y is not None:
|
|
ax.scatter(y[:, 0], y[:, 1])
|
|
for pair in matching_points:
|
|
ax.scatter([x[pair[0], 0], y[pair[1], 0]], [x[pair[0], 1], y[pair[1], 1]], s=50)
|
|
|
|
# get transformation from body to back contour
|
|
source = x[matching_points[:, 0], :]
|
|
destination = y[matching_points[:, 1], :]
|
|
trans = tps.TPS(source, destination)
|
|
transformed = trans(data_body.f.Vt[back, :])
|
|
|
|
ax = fig.add_subplot(122)
|
|
ax.set_xlim(0, 1)
|
|
ax.set_ylim(0, 1)
|
|
ax.set_aspect('equal')
|
|
ax.scatter(transformed[:, 0], transformed[:, 1])
|
|
if y is not None:
|
|
ax.scatter(y[:, 0], y[:, 1])
|
|
for pair in matching_points:
|
|
ax.scatter([transformed[pair[0], 0], y[pair[1], 0]], [transformed[pair[0], 1], y[pair[1], 1]], s=50)
|
|
|
|
plt.show()
|
|
|
|
data_back_high = np.load("Blender/uv_data_plane_1_high_res.npz", allow_pickle=True)
|
|
y_high = data_back_high.f.Vt
|
|
# y_high[:, 0] = y_high[:, 0] + 0.1
|
|
# y_high[:, 1] = y_high[:, 1] + 0.42
|
|
|
|
closest_points = get_closest_points_normals(y_high, transformed)
|
|
# closest_points = get_closest_point(y_high, transformed)
|
|
unique_mask = np.zeros_like(closest_points, dtype=bool)
|
|
unique_mask[np.unique(closest_points, return_index=True)[1]] = True
|
|
matching_points = np.array([closest_points[unique_mask], np.array(range(y_high.shape[0]))[unique_mask]]).T
|
|
# matching_points_2 = []
|
|
# for i, p in enumerate(matching_points):
|
|
# if i % 10 == 0:
|
|
# matching_points_2.append(p)
|
|
# matching_points = np.array(matching_points_2)
|
|
|
|
source_2 = transformed[matching_points[:, 0], :]
|
|
destination_2 = y_high[matching_points[:, 1], :]
|
|
trans_2 = tps.TPS(source_2, destination_2)
|
|
transformed_2 = trans_2(transformed)
|
|
|
|
|
|
|
|
fig = plt.figure(figsize=(15, 15))
|
|
ax = fig.add_subplot(121)
|
|
ax.set_xlim(0, 1)
|
|
ax.set_ylim(0, 1)
|
|
ax.set_aspect('equal')
|
|
ax.scatter(source_2[:, 0], source_2[:, 1])
|
|
ax.scatter(destination_2[:, 0], destination_2[:, 1])
|
|
for i, pair in enumerate(zip(source_2, destination_2)):
|
|
if i % 5 == 0:
|
|
s, d = pair
|
|
ax.scatter([s[0], d[0]], [s[1], d[1]])
|
|
|
|
ax = fig.add_subplot(122)
|
|
ax.set_xlim(0, 1)
|
|
ax.set_ylim(0, 1)
|
|
ax.set_aspect('equal')
|
|
ax.scatter(transformed_2[:, 0], transformed_2[:, 1])
|
|
ax.scatter(y_high[:, 0], y_high[:, 1])
|
|
plt.show()
|
|
|
|
return transformed_2
|
|
|
|
|
|
def match_back_to_front(front_uv):
|
|
# load data
|
|
data_body = np.load("Blender/uv_data_smpl_2.npz", allow_pickle=True)
|
|
# data_back = np.load("Blender/uv_data_plane_1_2.npz", allow_pickle=True)
|
|
|
|
# translate back uv to match body
|
|
# uv_back = data_back.f.Vt
|
|
# uv_back[:, 0] = uv_back[:, 0] + 0.1
|
|
# uv_back[:, 1] = uv_back[:, 1] + 0.42
|
|
|
|
# find front and back indices in uv
|
|
uv_verts = np.append(data_body.f.Vt, np.zeros((data_body.f.Vt.shape[0], 1)), axis=-1)
|
|
m = tm.Trimesh(uv_verts, data_body.f.Ft, process=False, validate=False)
|
|
front, back = tm.graph.connected_components(m.edges, min_len=1, nodes=None, engine=None)
|
|
|
|
# get mesh to uv mapping for the back vertices
|
|
m2u_body = mesh2UV(data_body.f.F, data_body.f.Ft)
|
|
m2u_body = filter_m2u(m2u_body, set(back))
|
|
m2u_body = convert_to_numbers(m2u_body)
|
|
|
|
# m2u_back = mesh2UV(data_back.f.F, data_back.f.Ft)
|
|
# m2u_back = convert_to_numbers(m2u_back)
|
|
|
|
# create mask for filtering vertices belonging to the back
|
|
body_back_mask = np.zeros((data_body.f.Vt.shape[0]), dtype=bool)
|
|
body_back_mask[back] = True
|
|
body_front_mask = np.zeros((data_body.f.Vt.shape[0]), dtype=bool)
|
|
body_front_mask[front] = True
|
|
|
|
# define which points should match after the transform on the mesh
|
|
# matching_points = np.array([[3284, 3], [9557, 13], [9278, 15], [8095, 1], [10597, 10], [10602, 17], [7114, 9], [6989, 16], [6983, 11], [4219, 0], [5628, 14], [5908, 12], [1447, 2], [1086, 5], [21, 8], [33, 4], [1888, 7], [2931, 6]])
|
|
|
|
# # map matching point to uv coordinates
|
|
# for i in range(matching_points.shape[0]):
|
|
# matching_points[i, :] = [get_masked_index(body_back_mask, m2u_body[matching_points[i,0]]), m2u_back[matching_points[i, 1]]]
|
|
|
|
m2u_body_all = mesh2UV(data_body.f.F, data_body.f.Ft)
|
|
m2u_body_all = m2uv_array(m2u_body_all)
|
|
border_pairs = m2u_body_all[:, 0] != m2u_body_all[:, 1]
|
|
border_pairs = m2u_body_all[border_pairs, :]
|
|
matching_points = border_pairs
|
|
matching_points_sorted = []
|
|
for p in matching_points:
|
|
if p[0] in front:
|
|
matching_points_sorted.append([p[0], p[1]])
|
|
elif p[0] in back:
|
|
matching_points_sorted.append([p[1], p[0]])
|
|
else:
|
|
assert False
|
|
# add neck V shape control point
|
|
matching_points_sorted.append([m2u_body_all[10228][0], m2u_body_all[33][0]])
|
|
matching_points = np.array(matching_points_sorted)
|
|
mm = matching_points.copy()
|
|
|
|
for i in range(matching_points.shape[0]):
|
|
matching_points[i, :] = [get_masked_index(body_back_mask, matching_points[i,1]), get_masked_index(body_front_mask, matching_points[i,0])]
|
|
|
|
temp = np.zeros_like(data_body.f.Vt)
|
|
temp[front, :] = front_uv
|
|
|
|
x = data_body.f.Vt[body_back_mask, :]
|
|
y = temp[body_front_mask, :] # front_uv #
|
|
|
|
fig = plt.figure(figsize=(15, 15))
|
|
ax = fig.add_subplot(121)
|
|
ax.set_xlim(0, 1)
|
|
ax.set_ylim(0, 1)
|
|
ax.set_aspect('equal')
|
|
ax.scatter(x[:, 0], x[:, 1])
|
|
if y is not None:
|
|
ax.scatter(y[:, 0], y[:, 1])
|
|
for pair in matching_points:
|
|
ax.scatter([x[pair[0], 0], y[pair[1], 0]], [x[pair[0], 1], y[pair[1], 1]], s=50)
|
|
|
|
# get transformation from body to back contour
|
|
source = x[matching_points[:, 0], :]
|
|
destination = y[matching_points[:, 1], :]
|
|
trans = tps.TPS(source, destination)
|
|
transformed = trans(data_body.f.Vt[back, :])
|
|
|
|
ax = fig.add_subplot(122)
|
|
ax.set_xlim(0, 1)
|
|
ax.set_ylim(0, 1)
|
|
ax.set_aspect('equal')
|
|
ax.scatter(transformed[:, 0], transformed[:, 1])
|
|
if y is not None:
|
|
ax.scatter(y[:, 0], y[:, 1])
|
|
for pair in matching_points:
|
|
ax.scatter([transformed[pair[0], 0], y[pair[1], 0]], [transformed[pair[0], 1], y[pair[1], 1]], s=50)
|
|
|
|
plt.show()
|
|
|
|
|
|
return transformed
|
|
|
|
|
|
def generate_mask():
|
|
import cv2
|
|
res = 1024
|
|
front = np.load("Blender/uv_data_plane_0_high_res_2.npz", allow_pickle=True)
|
|
back = np.load("Blender/uv_data_plane_1_high_res_2.npz", allow_pickle=True)
|
|
points = np.round(np.concatenate((front.f.Vt, back.f.Vt)) * res).astype(np.int32)
|
|
img = np.zeros((res, res, 3))
|
|
img[points[:, 0], points[:, 1], :] = 255
|
|
cv2.imwrite('MeshUtils/mask_1024.png', img)
|
|
|
|
# do paint now
|
|
|
|
img = cv2.imread('MeshUtils/mask_1024_2.png')
|
|
img = img[:, :, 0] == 255
|
|
np.save('MeshUtils/mask_1024.npy', img)
|
|
|
|
def generate_mask_2():
|
|
import cv2
|
|
res = 256
|
|
front = np.load("Blender/uv_data_plane_stretched_front_high_extra.npz", allow_pickle=True)
|
|
back = np.load("Blender/uv_data_plane_stretched_front_high_extra.npz", allow_pickle=True)
|
|
points = np.floor(np.concatenate((front.f.Vt, back.f.Vt + [1, 0])) * res).astype(np.int32)
|
|
img = np.zeros((res * 2, res, 3))
|
|
img[points[:, 0], points[:, 1], :] = 255
|
|
cv2.imwrite('MeshUtils/mask_256x512.png', img)
|
|
|
|
# do paint now
|
|
|
|
img = cv2.imread('MeshUtils/mask_256x512.png')
|
|
img = img[:, :, 0] > 240
|
|
np.save('MeshUtils/mask_256x512.npy', img)
|
|
|
|
pass
|
|
|
|
|
|
def generate_uv_mask_border():
|
|
import cv2
|
|
mask = np.load('MeshUtils/mask_256x512.npy')
|
|
uv_res = 256
|
|
mask_border = (cv2.resize(mask * 255.0, (uv_res, uv_res * 2)) / 255).astype(bool)[:, :, np.newaxis]
|
|
mask_border = cv2.erode(mask_border.astype(np.float32), cv2.getStructuringElement(cv2.MORPH_RECT, (2 * 2 + 1, 2 * 2 + 1), (2, 2)), iterations=1).astype(bool)
|
|
|
|
mask[mask_border == True] = 0
|
|
# plt.imshow(mask)
|
|
idx_front = np.transpose(np.vstack(np.nonzero(mask[:256, :])), [1,0])
|
|
|
|
test = np.zeros_like(mask)
|
|
test[idx_front[:,0], idx_front[:,1]] = 1
|
|
idx_back = idx_front.copy()
|
|
idx_back[:, 0] += 256
|
|
test[idx_back[:,0], idx_back[:,1]] = 1
|
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
def front():
|
|
# load data
|
|
data_body = np.load("custom_data/uv_map/uv_data_smpl_2.npz", allow_pickle=True)
|
|
data_front = np.load("custom_data/uv_map/uv_data_plane_stretched_front_low.npz", allow_pickle=True)
|
|
|
|
# translate back uv to match body
|
|
uv_front = data_front.f.Vt
|
|
# uv_back[:, 0] = uv_back[:, 0] + 0.1
|
|
# uv_back[:, 1] = uv_back[:, 1] + 0.42
|
|
|
|
# find front and back indices in uv
|
|
uv_verts = np.append(data_body.f.Vt, np.zeros((data_body.f.Vt.shape[0], 1)), axis=-1)
|
|
m = tm.Trimesh(uv_verts, data_body.f.Ft, process=False, validate=False)
|
|
front, back = tm.graph.connected_components(m.edges, min_len=1, nodes=None, engine=None)
|
|
|
|
# get mesh to uv mapping for the back vertices
|
|
m2u_body = mesh2UV(data_body.f.F, data_body.f.Ft)
|
|
m2u_body = filter_m2u(m2u_body, set(front))
|
|
m2u_body = convert_to_numbers(m2u_body)
|
|
|
|
m2u_front = mesh2UV(data_front.f.F, data_front.f.Ft)
|
|
m2u_front = convert_to_numbers(m2u_front)
|
|
|
|
# create mask for filtering vertices belonging to the back
|
|
body_front_mask = np.zeros((data_body.f.Vt.shape[0]), dtype=bool)
|
|
body_front_mask[front] = True
|
|
|
|
# define which points should match after the transform on the mesh
|
|
matching_points = np.array([ [9557, 12], [3284, 2], [3053, 5], [1888, 8], [10228, 4], [21, 7], [1209, 6], [1447, 3], [5908, 13], [4617, 15], [4219, 1], [6983, 10], [6989, 17], [7114, 9], [10602, 16], [10597, 11], [7884, 0], [8280, 14] ])
|
|
# matching_points = np.array([ [9557, 12], [3284, 2], [3053, 5], [1888, 8], [10228, 4], [21, 7], [1209, 6], [1447, 3], [5908, 13], [4793, 15], [4219, 1], [6983, 10], [6989, 17], [7114, 9], [10602, 16], [10597, 11], [7884, 0], [8450, 14], [13566, 18], [11767, 19], [8668, 20], [5012, 21], [12716, 22], [10917, 23], [8280, 24], [4617, 25], [2760, 26], [911, 27] ])
|
|
|
|
# map matching point to uv coordinates
|
|
for i in range(matching_points.shape[0]):
|
|
matching_points[i, :] = [get_masked_index(body_front_mask, m2u_body[matching_points[i,0]]), m2u_front[matching_points[i, 1]]]
|
|
|
|
x = data_body.f.Vt[body_front_mask, :]
|
|
y = uv_front
|
|
|
|
fig = plt.figure(figsize=(15, 15))
|
|
ax = fig.add_subplot(121)
|
|
# ax.set_xlim(0, 1)
|
|
# ax.set_ylim(0, 1)
|
|
ax.set_aspect('equal')
|
|
ax.scatter(x[:, 0], x[:, 1])
|
|
if y is not None:
|
|
ax.scatter(y[:, 0], y[:, 1])
|
|
for pair in matching_points:
|
|
ax.scatter([x[pair[0], 0], y[pair[1], 0]], [x[pair[0], 1], y[pair[1], 1]], s=50)
|
|
|
|
# get transformation from body to front contour
|
|
source = x[matching_points[:, 0], :]
|
|
destination = y[matching_points[:, 1], :]
|
|
trans = tps.TPS(source, destination)
|
|
transformed = trans(data_body.f.Vt[front, :])
|
|
|
|
# transformed = (transformed - transformed.min()) / (transformed.max() - transformed.min())
|
|
|
|
ax = fig.add_subplot(122)
|
|
# ax.set_xlim(0, 1)
|
|
# ax.set_ylim(0, 1)
|
|
ax.set_aspect('equal')
|
|
ax.scatter(transformed[:, 0], transformed[:, 1])
|
|
if y is not None:
|
|
ax.scatter(y[:, 0], y[:, 1])
|
|
for pair in matching_points:
|
|
ax.scatter([transformed[pair[0], 0], y[pair[1], 0]], [transformed[pair[0], 1], y[pair[1], 1]], s=50)
|
|
|
|
plt.show()
|
|
|
|
data_front_high = np.load("custom_data/uv_map/uv_data_plane_stretched_front_high.npz", allow_pickle=True)
|
|
y_high = data_front_high.f.Vt
|
|
# y_high[:, 0] = y_high[:, 0] + 0.1
|
|
# y_high[:, 1] = y_high[:, 1] + 0.42
|
|
|
|
closest_points = get_closest_points_normals(y_high, transformed)
|
|
# closest_points = get_closest_point(y_high, transformed)
|
|
unique_mask = np.zeros_like(closest_points, dtype=bool)
|
|
unique_mask[np.unique(closest_points, return_index=True)[1]] = True
|
|
matching_points = np.array([closest_points[unique_mask], np.array(range(y_high.shape[0]))[unique_mask]]).T
|
|
# matching_points_2 = []
|
|
# for i, p in enumerate(matching_points):
|
|
# if i % 10 == 0:
|
|
# matching_points_2.append(p)
|
|
# matching_points = np.array(matching_points_2)
|
|
|
|
source_2 = transformed[matching_points[:, 0], :]
|
|
destination_2 = y_high[matching_points[:, 1], :]
|
|
trans_2 = tps.TPS(source_2, destination_2)
|
|
transformed_2 = trans_2(transformed)
|
|
|
|
|
|
|
|
fig = plt.figure(figsize=(15, 15))
|
|
ax = fig.add_subplot(121)
|
|
ax.set_xlim(0, 1)
|
|
ax.set_ylim(0, 1)
|
|
ax.set_aspect('equal')
|
|
ax.scatter(source_2[:, 0], source_2[:, 1])
|
|
ax.scatter(destination_2[:, 0], destination_2[:, 1])
|
|
for i, pair in enumerate(zip(source_2, destination_2)):
|
|
if i % 5 == 0:
|
|
s, d = pair
|
|
ax.scatter([s[0], d[0]], [s[1], d[1]])
|
|
|
|
ax = fig.add_subplot(122)
|
|
ax.set_xlim(0, 1)
|
|
ax.set_ylim(0, 1)
|
|
ax.set_aspect('equal')
|
|
ax.scatter(transformed_2[:, 0], transformed_2[:, 1])
|
|
ax.scatter(y_high[:, 0], y_high[:, 1])
|
|
plt.show()
|
|
|
|
return transformed_2, mesh2UV(data_body.f.F, data_body.f.Ft), front, back
|
|
|
|
|
|
|
|
def faces2graph(F, v=None):
|
|
if v: v = set(v) # Only this verts
|
|
G = {}
|
|
for f in F:
|
|
for i in f:
|
|
if v and i not in v: continue
|
|
if not i in G:
|
|
G[i] = {}
|
|
for j in f:
|
|
if v and j not in v: continue
|
|
if i == j: continue
|
|
G[i][j] = 1
|
|
return G
|
|
|
|
|
|
def custom_smoothing(mesh, edges, pinned, F, iterations=10):
|
|
# custom_scatter(mesh[pinned, :]).show()
|
|
|
|
mesh_orig = mesh.copy()
|
|
G = faces2graph(F)
|
|
for _ in tqdm.tqdm(range(iterations)):
|
|
edge_lengths = np.sqrt(((mesh[edges[:,0], :] - mesh[edges[:,1], :]) ** 2 ).sum(-1))
|
|
|
|
edge_lengths_m = np.zeros((15068, 15068), np.float32)
|
|
edge_lengths_m[edges[:,0], edges[:,1]] = edge_lengths
|
|
edge_lengths_m[edges[:,1], edges[:,0]] = edge_lengths
|
|
|
|
# mean_edge = edge_lengths.mean()
|
|
|
|
# dist = cdist(mesh, mesh)
|
|
# for i in range(dist.shape[0]):
|
|
# dist[i,i] = 10000
|
|
|
|
# dist = sp.spatial.KDTree(mesh)
|
|
|
|
# closest_neighbour = edge_lengths_m.argmin(axis=0)
|
|
# if edge_lengths_m[i, closest_neighbour[i]] > mean_edge:
|
|
# continue
|
|
|
|
mesh_temp = mesh.copy()
|
|
for i in range(15068):
|
|
if i in pinned:
|
|
continue
|
|
|
|
# closest_neighbour = edge_lengths_m[i, :].argmin()
|
|
# if edge_lengths_m[i, closest_neighbour] > mean_edge:
|
|
# continue
|
|
|
|
neighbours = list(G[i].keys())
|
|
neighbours_dist = edge_lengths_m[i, neighbours]
|
|
weights = neighbours_dist / neighbours_dist.sum()
|
|
weights = np.ones_like(weights) / weights.shape[0]
|
|
mesh_temp[i, :] = np.sum(mesh[neighbours, :] * weights[:, np.newaxis], axis=0)
|
|
|
|
# neighbours_dist, neighbours = dist.query(mesh[i,:], k=10, workers=10)
|
|
# # if neighbours[0] == 15068:
|
|
# # continue
|
|
# neighbours_dist, neighbours = neighbours_dist[1:], neighbours[1:]
|
|
# weights = neighbours_dist / neighbours_dist.sum()
|
|
# mesh_temp[i, :] = np.sum(mesh[neighbours, :] * weights[:, np.newaxis], axis=0)
|
|
|
|
# mesh_temp[i, :] = np.mean(mesh[neighbours, :], axis=0)
|
|
|
|
# print()
|
|
|
|
# closest_neighbour = edge_lengths_m[i, :].argmin()
|
|
# if edge_lengths_m[i, closest_neighbour] < mean_edge:
|
|
# vector = mesh[i,:] - mesh[closest_neighbour,:]
|
|
# vector = vector / np.sqrt(np.sum(vector[0] ** 2 + vector[1] ** 2 ))
|
|
# to_move = vector / 1000
|
|
# # to_move = vector / edge_lengths_m[i, closest_neighbour] * 100
|
|
# mesh[i, :] -= to_move
|
|
|
|
mesh = mesh_temp
|
|
|
|
|
|
custom_scatter(mesh_orig, mesh).show()
|
|
print()
|
|
|
|
return mesh
|
|
|
|
def custom_scatter(x, y=None):
|
|
data = [go.Scatter(
|
|
x=x[:,0],
|
|
y=x[:,1],
|
|
mode='markers',
|
|
marker=dict(
|
|
size=3,
|
|
line=dict(width=0)
|
|
),
|
|
),]
|
|
|
|
if y is not None:
|
|
data.append(
|
|
go.Scatter(
|
|
x=y[:,0],
|
|
y=y[:,1],
|
|
mode='markers',
|
|
marker=dict(
|
|
size=3,
|
|
line=dict(width=0)
|
|
),
|
|
),)
|
|
|
|
fig = go.Figure(data=data)
|
|
|
|
# fig.update_layout(scene_aspectmode='manual',
|
|
# scene_aspectratio=dict(x=1, y=1, z=1))
|
|
# fig.update_layout(
|
|
# scene = dict(
|
|
# xaxis = dict(nticks=4, range=[-1,1],),
|
|
# yaxis = dict(nticks=4, range=[-1,1],),
|
|
# zaxis = dict(nticks=4, range=[-1,1],),)
|
|
# )
|
|
|
|
return fig
|
|
|
|
|
|
def smoothing():
|
|
data_body = np.load("custom_data/uv_map/uv_data_smpl_2.npz", allow_pickle=True)
|
|
uv_map = np.load('custom_data/uv_map/stretched_uv_map_3.npy')
|
|
F = np.load("custom_utils/smpl_faces.npy").astype(np.int32)
|
|
verts = np.concatenate((uv_map, np.ones((15068, 1))), axis=-1)
|
|
m = tm.Trimesh(verts, data_body.f.Ft, process=False, validate=False)
|
|
|
|
|
|
m2u_body_all = mesh2UV(data_body.f.F, data_body.f.Ft)
|
|
m2u_body_all = m2uv_array(m2u_body_all)
|
|
border_pairs = m2u_body_all[:, 0] != m2u_body_all[:, 1]
|
|
border_pairs = m2u_body_all[border_pairs, :]
|
|
pinned_vertices = border_pairs.reshape(-1)
|
|
right_legs_bottom = np.nonzero(((verts[:,0] > 1).astype(np.int8) * (verts[:,0] < 1.03).astype(np.int8)).astype(bool))[0]
|
|
left_legs_bottom = np.nonzero(verts[:,0] < 0.025)[0]
|
|
top_arms = np.nonzero(verts[:,1] > 0.995)[0]
|
|
bottom_arms = np.nonzero(verts[:,1] < 0.005)[0]
|
|
pinned_vertices = np.concatenate((pinned_vertices, right_legs_bottom, left_legs_bottom, top_arms, bottom_arms))
|
|
|
|
x = custom_smoothing(np.asarray(m.vertices)[:,:2], np.asarray(m.edges), pinned_vertices, data_body.f.Ft)
|
|
# m = tm.Trimesh(x, data_body.f.Ft, process=False, validate=False)
|
|
|
|
lap = tm.smoothing.laplacian_calculation(m, equal_weight=False, pinned_vertices=pinned_vertices)
|
|
# tm.smoothing.filter_laplacian(m, lamb=0.5, implicit_time_integration=False, iterations=1000, volume_constraint=False, laplacian_operator=lap)
|
|
# tm.smoothing.filter_mut_dif_laplacian(m, lamb=0.9, iterations=100, volume_constraint=True, laplacian_operator=lap)
|
|
tm.smoothing.filter_humphrey(m, iterations=1000, laplacian_operator=lap, alpha=0.01, beta=1.0)
|
|
# tm.smoothing.filter_taubin(m, iterations=25, lamb=0.01, nu=0.0015, laplacian_operator=lap)
|
|
|
|
|
|
x = uv_map
|
|
y = m.vertices[:, :2]
|
|
|
|
|
|
|
|
fig = go.Figure(data=[
|
|
# go.Scatter(
|
|
# x=x[:,0],
|
|
# y=x[:,1],
|
|
# mode='markers',
|
|
# marker=dict(
|
|
# size=3,
|
|
# line=dict(width=0)
|
|
# ),
|
|
# ),
|
|
go.Scatter(
|
|
x=y[:,0],
|
|
y=y[:,1],
|
|
mode='markers',
|
|
marker=dict(
|
|
size=3,
|
|
line=dict(width=0)
|
|
),
|
|
),
|
|
])
|
|
|
|
fig.update_layout(scene_aspectmode='manual',
|
|
scene_aspectratio=dict(x=1, y=1, z=1))
|
|
fig.update_layout(
|
|
scene = dict(
|
|
xaxis = dict(nticks=4, range=[-1,1],),
|
|
yaxis = dict(nticks=4, range=[-1,1],),
|
|
zaxis = dict(nticks=4, range=[-1,1],),)
|
|
)
|
|
|
|
fig.show()
|
|
|
|
dist = cdist(y, y)
|
|
for i in range(dist.shape[0]):
|
|
dist[i,i] = 10000
|
|
|
|
res = 1 / dist.min()
|
|
|
|
print()
|
|
|
|
|
|
pass
|
|
|
|
|
|
def front_3dto2d():
|
|
# load data
|
|
data_body = np.load("custom_data/uv_map/uv_data_smpl_2.npz", allow_pickle=True)
|
|
data_front = np.load("custom_data/uv_map/uv_data_plane_stretched_front_low.npz", allow_pickle=True)
|
|
|
|
laplacian_loss = sio.loadmat("custom_utils/ssmpl0.mat")['L'].tocoo()
|
|
edges = np.stack(np.nonzero(laplacian_loss.todense())).transpose()
|
|
|
|
# translate back uv to match body
|
|
uv_front = data_front.f.V
|
|
# uv_back[:, 0] = uv_back[:, 0] + 0.1
|
|
# uv_back[:, 1] = uv_back[:, 1] + 0.42
|
|
|
|
# find front and back indices in uv
|
|
uv_verts = np.append(data_body.f.Vt, np.zeros((data_body.f.Vt.shape[0], 1)), axis=-1)
|
|
m = tm.Trimesh(uv_verts, data_body.f.Ft, process=False, validate=False)
|
|
front, back = tm.graph.connected_components(m.edges, min_len=1, nodes=None, engine=None)
|
|
|
|
# get mesh to uv mapping for the back vertices
|
|
m2u_body = mesh2UV(data_body.f.F, data_body.f.Ft)
|
|
m2u_body = filter_m2u(m2u_body, set(front))
|
|
m2u_body = convert_to_numbers(m2u_body)
|
|
|
|
m2u_front = mesh2UV(data_front.f.F, data_front.f.Ft)
|
|
m2u_front = convert_to_numbers(m2u_front)
|
|
|
|
# create mask for filtering vertices belonging to the back
|
|
body_front_mask = np.zeros((data_body.f.Vt.shape[0]), dtype=bool)
|
|
body_front_mask[front] = True
|
|
|
|
# define which points should match after the transform on the mesh
|
|
matching_points = np.array([ [9557, 12], [3284, 2], [3053, 5], [1888, 8], [10228, 4], [21, 7], [1209, 6], [1447, 3], [5908, 13], [4617, 15], [4219, 1], [6983, 10], [6989, 17], [7114, 9], [10602, 16], [10597, 11], [7884, 0], [8280, 14] ])
|
|
|
|
# for i in range(matching_points.shape[0]):
|
|
# matching_points[i, :] = [get_masked_index(body_front_mask, matching_points[i,0]),matching_points[i, 1]]
|
|
|
|
|
|
x = data_body.f.V
|
|
# x = data_body.f.V[body_front_mask, :]
|
|
y = uv_front
|
|
|
|
# get transformation from body to front contour
|
|
source = x[matching_points[:, 0], :]
|
|
destination = y[matching_points[:, 1], :]
|
|
trans = TPS(source, destination)
|
|
transformed = trans(data_body.f.V)
|
|
|
|
z = transformed
|
|
fig = go.Figure(data=[
|
|
# go.Scatter3d(
|
|
# x=x[:,0],
|
|
# y=x[:,1],
|
|
# z=x[:,2],
|
|
# ),
|
|
go.Scatter3d(
|
|
x=y[:,0],
|
|
y=y[:,1],
|
|
z=y[:,2],
|
|
mode='markers',
|
|
marker=dict(
|
|
size=10,
|
|
line=dict(width=0)
|
|
),
|
|
),
|
|
go.Scatter3d(
|
|
x=z[:,0],
|
|
y=z[:,1],
|
|
z=z[:,2],
|
|
mode='markers',
|
|
marker=dict(
|
|
size=10,
|
|
line=dict(width=0)
|
|
),
|
|
|
|
),
|
|
])
|
|
|
|
fig.update_layout(scene_aspectmode='manual',
|
|
scene_aspectratio=dict(x=1, y=1, z=1))
|
|
fig.update_layout(
|
|
scene = dict(
|
|
xaxis = dict(nticks=4, range=[-1,1],),
|
|
yaxis = dict(nticks=4, range=[-1,1],),
|
|
zaxis = dict(nticks=4, range=[-1,1],),)
|
|
)
|
|
|
|
fig.show()
|
|
|
|
print()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
# generate_mask_2()
|
|
smoothing()
|
|
# generate_uv_mask_border()
|
|
|
|
# front_uv, m2uv, front_mask, back_mask = front_3dto2d()
|
|
|
|
|
|
# back_uv = match_back_to_front(front_uv)
|
|
# # back_uv = back()
|
|
|
|
# final = np.zeros((15068, 2))
|
|
# final[front_mask] = front_uv
|
|
# final[back_mask] = back_uv + [1, 0]
|
|
# np.save('MeshUtils\stretched_uv_map_3.npy', final)
|
|
|
|
pass |