69 lines
2.1 KiB
Python
69 lines
2.1 KiB
Python
import os
|
|
import sys
|
|
|
|
import torch
|
|
import torch.nn as nn
|
|
from torch.nn import functional as F
|
|
|
|
sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/..')
|
|
from models.convnext import ConvNeXt, CNBlockConfig
|
|
|
|
class ViewEncoder(nn.Module):
|
|
def __init__(self, features=128, device='cuda'):
|
|
super().__init__()
|
|
self.features: int = features
|
|
|
|
self.flatten = nn.Flatten()
|
|
self.lin1 = nn.LazyLinear(features).to(device)
|
|
self.act1 = nn.GELU().to(device)
|
|
self.lin2 = nn.Linear(features, features * 15 * 20).to(device)
|
|
self.act2 = nn.Tanh().to(device)
|
|
|
|
|
|
def forward(self, input: torch.Tensor) -> torch.Tensor:
|
|
x: torch.Tensor = self.flatten(input)
|
|
x= self.lin1(x)
|
|
x = self.act1(x)
|
|
x = self.lin2(x)
|
|
x = self.act2(x)
|
|
x = x.view(x.shape[0],self.features,15,20)
|
|
|
|
return x
|
|
|
|
class DepthEstimation(nn.Module):
|
|
def __init__(self, view_features=128, device='cuda', use_skip=False):
|
|
super().__init__()
|
|
|
|
|
|
block_setting = [
|
|
CNBlockConfig(96, 192, 3),
|
|
CNBlockConfig(192, 384, 3),
|
|
CNBlockConfig(384, 768, 9),
|
|
CNBlockConfig(768, None, 3),
|
|
]
|
|
|
|
block_setting_reverse = [
|
|
CNBlockConfig(768 + view_features, 384, 3),
|
|
CNBlockConfig(384 * ( 2 if use_skip else 1), 192, 9),
|
|
CNBlockConfig(192 * ( 2 if use_skip else 1), 96, 3),
|
|
CNBlockConfig(96 * ( 2 if use_skip else 1), 27, 3),
|
|
CNBlockConfig(27, 9, 3),
|
|
CNBlockConfig(9, None, 3),
|
|
]
|
|
|
|
self.convnext = ConvNeXt(block_setting, device=device)
|
|
self.flatten = nn.Flatten()
|
|
self.decoder = ConvNeXt(block_setting_reverse, use_skip=use_skip, reversed=True, device=device)
|
|
|
|
self.view_encoder = ViewEncoder(view_features)
|
|
|
|
|
|
def forward(self, input) -> torch.Tensor:
|
|
image, view = input
|
|
x, features = self.convnext(image)
|
|
view_features = self.view_encoder(view)
|
|
# x = self.flatten(x)
|
|
x = self.decoder([torch.concat((x, view_features), axis=1),features])
|
|
|
|
return x
|