cvpr/models/base.py

import tensorflow as tf
from convnext.models.convnext_tf import ConvNeXt, ConvNeXtReverse, Block, ConvNeXtReverseLabels

class Bottleneck(tf.keras.Model):
    
    def __init__(self, dim, latent_dim, kernel_size=7):
        super(Bottleneck, self).__init__()

        self.bottleneck = tf.keras.Sequential(
                [Block(dim=dim, drop_path=0, kernel_size=kernel_size,
                       layer_scale_init_value=1e-6, prefix=f'bottleneck{j}') for j in range(5)]
            )
            
        self.flatten = tf.keras.layers.Flatten()
        self.dense = tf.keras.layers.Dense(latent_dim)

    def call(self, inputs, training):
        x = inputs

        x = self.bottleneck(x, training=training)
        x = self.flatten(x)
        x = self.dense(x, training=training)

        return x


class VariationalEncoder(tf.keras.Model):
    
    def __init__(self, latent_dim, depths, dims, kernel_size=7):
        super(VariationalEncoder, self).__init__()

        self.encoder = ConvNeXt(depths=depths, dims=dims, include_top=False, kernel_size=kernel_size)

        self.flatten = tf.keras.layers.Flatten()
        self.mean = tf.keras.layers.Dense(latent_dim)
        self.log_var = tf.keras.layers.Dense(latent_dim)

    def call(self, inputs, training):
        x = inputs

        x = self.encoder(x, training=training)
        x = self.flatten(x)
        mean = self.mean(x)
        log_var = self.log_var(x)

        return mean, log_var


class AEEncoder(tf.keras.Model):
    
    def __init__(self, latent_dim, depths, dims, bottleneck_size=32, kernel_size=7):
        super(AEEncoder, self).__init__()

        self.encoder = ConvNeXt(depths=depths, dims=dims, include_top=False, kernel_size=kernel_size)
        self.bottleneck = Bottleneck(bottleneck_size, latent_dim, kernel_size)


    def call(self, inputs, training):
        if len(inputs) == 2:
            x, _ = inputs
        else:
            x = inputs

        x = self.encoder(x, training=training)
        x = self.bottleneck(x, training=training)

        if len(inputs) == 2:
            return x, inputs[1]
        else:
            return x


class Decoder(tf.keras.Model):
    
    def __init__(self, input_size, depths, dims, kernel_size=7, is_label=False, upscale=False):
        super(Decoder, self).__init__()

        self.dense = tf.keras.layers.Dense(tf.reduce_prod(input_size))
        self.reshape = tf.keras.layers.Reshape(input_size)
        self.upscale = upscale
        
        if not is_label:
            self.decoder = ConvNeXtReverse(depths=list(reversed(depths)), dims=list(reversed(dims)), kernel_size=kernel_size)
        else:
            self.decoder = ConvNeXtReverseLabels(depths=list(reversed(depths)), dims=list(reversed(dims)), nr_of_classes=7)
            

    def call(self, inputs, training):
        if len(inputs) == 2:
            x, mask = inputs
        else:
            x = inputs

        x = self.dense(x)
        x = self.reshape(x)
        x = self.decoder(x, training=training)

        if self.upscale:
            x = tf.image.resize(x, size=(512, 256), method=tf.image.ResizeMethod.LANCZOS5)


        if len(inputs) == 2:
            if len(mask.shape) == 2:
                x = x * mask[tf.newaxis,:,:,tf.newaxis]
            else:
                x = x * mask[:,:,:,tf.newaxis]

        return x



class VariationalEncoderNew(tf.keras.Model):
    
    def __init__(self, depths, dims, kernel_size=7):
        super(VariationalEncoderNew, self).__init__()

        self.encoder = ConvNeXt(depths=depths, dims=dims, include_top=False, kernel_size=kernel_size)

        self.mean_branch = tf.keras.Sequential(
                [Block(dim=64, drop_path=0, kernel_size=kernel_size,
                       layer_scale_init_value=1e-6, prefix=f'bottleneck{j}') for j in range(5)]
            )

        self.var_branch = tf.keras.Sequential(
                [Block(dim=64, drop_path=0, kernel_size=kernel_size,
                       layer_scale_init_value=1e-6, prefix=f'bottleneck{j}') for j in range(5)]
            )

        self.flatten_mean = tf.keras.layers.Flatten()
        self.flatten_var = tf.keras.layers.Flatten()



    def call(self, inputs, training):
        x = inputs

        x = self.encoder(x, training=training)
        mean = self.mean_branch(x)
        log_var = self.var_branch(x)

        mean = self.flatten_mean(mean)
        log_var = self.flatten_var(log_var)

        return mean, log_var
        


class AEEncoderNew(tf.keras.Model):
    
    def __init__(self, depths, dims, kernel_size=7, activation='tanh'):
        super(AEEncoderNew, self).__init__()

        self.encoder = ConvNeXt(depths=depths, dims=dims, include_top=False, kernel_size=kernel_size)
        self.act = tf.keras.layers.Activation(activation)


    def call(self, inputs, training):
        if len(inputs) == 2:
            x, _ = inputs
        else:
            x = inputs

        x = self.encoder(x, training=training)
        x = self.act(x)

        if len(inputs) == 2:
            return x, inputs[1]
        else:
            return x


class DecoderNew(tf.keras.Model):
    
    def __init__(self, depths, dims, kernel_size=7, is_label=False):
        super(DecoderNew, self).__init__()

        self.is_label = is_label

        if not is_label:
            self.decoder = ConvNeXtReverse(depths=depths, dims=dims, kernel_size=kernel_size)
        else:
            self.decoder = ConvNeXtReverseLabels(depths=depths, dims=dims, nr_of_classes=7)
            

    def call(self, inputs, training):
        if len(inputs) == 2:
            x, mask = inputs
        else:
            x = inputs

        x = self.decoder(x, training=training)

        # if self.is_label:

        if len(inputs) == 2:
            if len(mask.shape) == 2:
                x = x * mask[tf.newaxis,:,:,tf.newaxis]
            else:
                x = x * mask[:,:,:,tf.newaxis]

        return x