working pyramid pipeline

2022-09-27 21:26:15 +00:00
parent 1da83294ee
commit fa8f24f0c5
10 changed files with 997 additions and 213 deletions
@@ -0,0 +1,97 @@
+import os
+import time
+import random
+import tensorflow as tf
+import numpy as np
+import scipy as sp
+# import plotly.graph_objects as go
+# import plotly.express as ex
+
+from custom_utils.custom_utils import load_model
+from models.base import AEEncoder, VariationalEncoder, Decoder
+from models.upscaler import UpscalerBasic
+
+
+class PyramidLevel(tf.keras.Model):
+    
+    def __init__(self, config):
+        super(PyramidLevel, self).__init__()
+        self.latent_dim = config['latent_dim']
+        self.size = config['resolution']
+
+        self.encoder = VariationalEncoder(config['latent_dim'], config['encoder_depths'], config['encoder_dims'], kernel_size=config['kernel_size'])
+        self.decoder = Decoder(config['decoder_input'], config['decoder_depths'], config['decoder_dims'], config['kernel_size'])
+
+        self.upscale_enhancer = None
+        if config['upscale_enhancer'] != '':
+            self.upscale_enhancer = UpscalerBasic(512, config['resolution'])
+            load_model(self.upscale_enhancer, config['upscale_enhancer'])
+            self.upscale_enhancer = tf.keras.Sequential([
+                self.upscale_enhancer.flatten,
+                self.upscale_enhancer.seq,
+                self.upscale_enhancer.reshape
+                ])
+
+        if config['avg_garment_path'] != '':
+            self.avg_garment = np.load(config['avg_garment_path'])
+        else:
+            self.avg_garment = None
+
+
+    def enhance(self, low, mask, training=None):
+        x = self.upscale_enhancer(low, training=training)
+        x = x * (1 - mask[:,:,:,tf.newaxis])
+        low = x + low
+        return low
+
+
+    def sample(self, z_cond, uv_mask, noise=None):
+        if noise is None:
+            eps = tf.random.normal(shape=(uv_mask.shape[0], self.latent_dim))
+        else:
+            eps = noise
+
+        z = tf.concat([eps, z_cond], axis=-1)
+
+        x = self.decoder(z)
+        
+        if self.avg_garment is not None:
+            x = x + self.avg_garment[tf.newaxis,:,:,:]
+        
+        x = x * uv_mask[:,:,:,tf.newaxis]
+
+        return x
+
+
+    def reparameterize(self, mean, logvar, fix_noise=False):
+        if not fix_noise:
+            eps = tf.random.normal(shape=mean.shape)
+        else:
+            eps = tf.random.normal(shape=(1, mean.shape[1]))
+            eps = tf.tile(eps, [mean.shape[0],1])
+        return eps * tf.exp(logvar * .5) + mean
+
+
+    def call(self, inputs, z_cond, training=None):
+        uv_unposed, uv_template, uv_body_posed, uv_curv, uv_mask, uv_mask_high = inputs
+
+        x = tf.concat([uv_unposed, uv_template, uv_body_posed, uv_curv], axis=-1)
+        mean, logvar = self.encoder(x, training=training)
+        z = self.reparameterize(mean, logvar)
+
+        z = tf.concat([z, z_cond], axis=-1)
+        x = self.decoder(z, training=training)
+
+        if self.avg_garment is not None:
+            x = x + self.avg_garment[tf.newaxis,:,:,:]
+
+        x = x * uv_mask[:,:,:,tf.newaxis]
+
+        if self.upscale_enhancer is not None:
+            pred_low = self.enhance(x, uv_mask, training=training)
+            x = tf.image.resize(pred_low, (512, 256), tf.image.ResizeMethod.LANCZOS5)
+        
+        x = x * uv_mask_high[:,:,:,tf.newaxis]
+
+        return x, mean, logvar
+