invetigate registration errors

custom_utils/custom_utils.py

@@ -107,7 +107,7 @@ class UV2MeshLabel():
         return self.uv_to_mesh(uv)
 
 
-def _maskFaces(V, F, mask):
+def _maskFaces(V, F, mask, return_unfixed=False):
     V_mask = mask
     F_mask = np.take(V_mask, F.reshape(-1), axis=0)
     F_mask = F_mask.reshape((-1, 3)).sum(axis=1) == 3
@@ -117,6 +117,9 @@ def _maskFaces(V, F, mask):
     F_new = np.take(V_ind, F[F_mask, :].reshape(-1), axis=0).reshape((-1, 3))
     V_new = V[V_mask, :]
 
+    if return_unfixed:
+        return V_new, F_new, F[F_mask, :]
+    else:
         return V_new, F_new
 
 

data/cloth3d_io.py

@@ -260,9 +260,12 @@ def read_posed_garment(folder, id, garment, map=None, mask=None, shape=None, sha
     return verts
 
 
-def read_garment_mask(folder, id, garment):
+def read_garment_mask(folder, id, garment, return_raw=False):
     raw_mask = sio.loadmat(os.path.join(folder, id, garment + '_mask.mat'))['mask'].reshape((-1))
+    if not return_raw:
         mask = raw_mask > 0
+    else:
+        mask = raw_mask
     return mask
 
 

data/investigate.py

@@ -3,6 +3,7 @@ import sys
 import random
 import numpy as np
 import plotly.graph_objects as go
+from pymeshfix import MeshFix
 
 sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/..')
 from data.cloth3d_io import read_garment_mask, read_garment_template, read_posed_garment_frame, loadInfo, read_posed_body_frame, read_posed_body, read_posed_garment, read_curvatures, read_curvature_frame, writeOBJ
@@ -51,7 +52,7 @@ def build_lookup_table(repeated):
             lookup[repeated[i][j]] = [repeated[i][k] for k in range(len(repeated[i]))] # if k != j
     return lookup
 
-def mask_faces(all_faces, mask, repeated, verts):
+def mask_faces(all_faces, mask, raw_mask, repeated, verts):
     # if any of the vertices of a face is masked, mask the face
     lookup = build_lookup_table(repeated)
     faces = []
@@ -79,6 +80,30 @@ def mask_faces(all_faces, mask, repeated, verts):
                                     if mask[v1[i]] and mask[v2[j]] and mask[v3[k]]:
                                         # if the euclidean distance between any two of the vertices of the face is less than 0.05 then append the face
                                         # if np.linalg.norm(verts[v1[i]] - verts[v2[j]]) < 0.05 and np.linalg.norm(verts[v1[i]] - verts[v3[k]]) < 0.05 and np.linalg.norm(verts[v2[j]] - verts[v3[k]]) < 0.05:
+                                        
+                                        #only add the face if this face originally existed among the faces of the garment
+                                        # if not np.any([np.array([v1[i], v2[j], v3[k]], dtype=np.int32) == face for face in all_faces]):
+                                        #     continue
+                                        
+                                        # if all vertices of the face have a value of 2 in the raw mask then don't add the face
+                                        # if np.all(raw_mask[[v1[i], v2[j], v3[k]]] == 2):
+                                        #     continue
+
+                                        # skip face if it's area is more than 0.001 and at least one of the vertices has value of 2 in the raw mask
+                                        # if np.linalg.norm(np.cross(verts[v1[i]] - verts[v2[j]], verts[v1[i]] - verts[v3[k]])) > 0.001 and np.any(raw_mask[[v1[i], v2[j], v3[k]]] == 2):
+                                        #     continue
+                                        
+                                        # if np.linalg.norm(np.cross(verts[v1[i]] - verts[v2[j]], verts[v1[i]] - verts[v3[k]])) > 0.001:
+                                        #     continue
+
+                                        # if any edge of the face is longer than 0.1 and both vertices of the edge have a value of 2 in the raw mask then skip the face
+                                        # if np.linalg.norm(verts[v1[i]] - verts[v2[j]]) > 0.05 and (raw_mask[v1[i]] == 2 or raw_mask[v2[j]] == 2):
+                                        #     continue
+                                        # if np.linalg.norm(verts[v1[i]] - verts[v3[k]]) > 0.05 and (raw_mask[v1[i]] == 2 or raw_mask[v3[k]] == 2):
+                                        #     continue
+                                        # if np.linalg.norm(verts[v2[j]] - verts[v3[k]]) > 0.05 and (raw_mask[v2[j]] == 2 or raw_mask[v3[k]] == 2):
+                                        #     continue
+
                                         faces.append(np.array([v1[i], v2[j], v3[k]], dtype=np.int32))
                                         assert np.all(verts[face] == verts[faces[-1]])
                                         found_replacement = True
@@ -100,22 +125,38 @@ def check(folder, frame):
     id, frame_nr, garment = frame.split()
     frame_nr = int(frame_nr)
     info = loadInfo(os.path.join(folder, id, 'info.mat'))
-    mask = read_garment_mask(folder, id, garment)
+    raw_mask = read_garment_mask(folder, id, garment, return_raw=True)
+    mask = raw_mask > 0
     mesh_posed = read_posed_garment_frame(folder, id, garment, frame_nr, mask=mask, shape=info['shape'], pose=info['poses'][:,frame_nr], deorient=use_no_orient, unshape=no_shape)
 
+
     v = mesh_posed[mask,:]
     is_skirt = (garment == 'Dress' or garment == 'Skirt')
     skirt_mod = '_skirt' if is_skirt else ''
     F = np.load(f'custom_utils/smpl_faces{skirt_mod}.npy').astype(np.int32)
-    _, faces = _maskFaces(np.zeros((14475,3)), F, mask)
+    _, faces, unfixed = _maskFaces(np.zeros((14475,3)), F, mask, True)
 
+    meshfix = MeshFix(v, faces)
+    holes = meshfix.extract_holes()
     # plot(v, faces)
 
     unq, idx, count = np.unique(mesh_posed, axis=0, return_counts=True, return_index=True)
     repeated_groups = unq[count > 1]
 
+    new_border = []
+    for border in np.array(holes.points):
+        # find the index of the vertex in the original mesh
+        ind = np.argwhere(np.all(mesh_posed == border, axis=1))
+        new_border.extend(ind.ravel())
+
+    raw_mask[new_border] = 2    
     mask_2 = np.zeros((14475), dtype=bool)
+    mask_2_raw = np.zeros((14475), dtype=np.uint8)
     mask_2[idx] = True
+    mask_2_raw = mask_2.astype(np.uint8)
+    mask_3 = np.zeros((14475), dtype=bool)
+    mask_3[mask_2] = raw_mask[mask_2] == 2
+    mask_2_raw[mask_3] = 2
     v_2 = mesh_posed[mask_2,:]
     _, faces_2 = _maskFaces(np.zeros((14475,3)), F, mask_2)
     # plot(v_2, faces_2)
@@ -127,11 +168,46 @@ def check(folder, frame):
         repeated_idx = np.argwhere(np.all(mesh_posed == repeated_group, axis=1))
         repeated.append(repeated_idx.ravel())
 
-    faces_3 = mask_faces(F, mask_2, repeated, mesh_posed)
+    # # if a vertex from v_2 is not part of v then set the corresponding value in mask_0 to 0
-    writeOBJ('temp/investigate_3.obj', v_2, faces_3)
+
+    # mask_0 = np.zeros((v_2.shape[0]), dtype=bool)
+    # for i, _v in enumerate(v_2):
+    #     if not np.any([np.all(_v == _w) for _w in v]):
+    #         mask_0[i] = True
+    # mask_0_0 = np.zeros((14475), dtype=bool)
+    # mask_0_0[mask_2] = 1 - mask_0
+    # mask_4 = np.logical_not(mask_0)
+
+    faces_3 = mask_faces(F, mask_2, mask_2_raw, repeated, mesh_posed)
+    # writeOBJ('temp/investigate_3.obj', v_2, faces_3)
     plot(v_2, faces_3)
 
 
+    # plot repeated
+    r = []
+    for rr in repeated:
+        r.extend(rr)
+    x = mesh_posed
+    y = mesh_posed[r,:]
+    go.Figure(data = [go.Scatter3d(x=x[:, 0], y=x[:, 1], z=x[:, 2], marker=dict(size=3, line=dict(width=0),), mode='markers'),
+                      go.Scatter3d(x=y[:, 0], y=y[:, 1], z=y[:, 2], marker=dict(size=5, line=dict(width=0),), mode='markers'),
+                     ]).show()
+
+
+    # unq, idx, count = np.unique(v, axis=0, return_counts=True, return_index=True)
+    # repeated_groups = unq[count > 1]
+    # repeated = []
+    # for repeated_group in repeated_groups:
+    #     repeated_idx = np.argwhere(np.all(v == repeated_group, axis=1))
+    #     repeated.append(repeated_idx.ravel())
+
+    # mask_3 = np.zeros((v.shape[0]), dtype=bool)
+    # mask_3[idx] = True
+
+    # faces_4 = mask_faces(faces.astype(np.int64), mask_3, mask_2_raw, repeated, v)
+    # plot(v[mask_3,:], faces_4)
+
+
     pass