PyTorch Image transformation with gridsampler, weird behaviour while optimizing grid

Question

Im trying to get the per-pixel transformation to fit one image (+ background) to a result. Background Image + Input image should be transformed into desired result

to achieve this i am using the PyTorch gridsampler and autograd to optimize the grid. The transformed input will be added to the unchanged background.


ToTensor = torchvision.transforms.ToTensor()
FromTensor = torchvision.transforms.ToPILImage()

backround= ToTensor(Image.open("backround.png"))
pic = ToTensor(Image.open("pic.png"))
goal = ToTensor(Image.open("goal.png"))

empty = empty.expand(1, 3, empty.size()[1], empty.size()[2])
pic = pic.expand(1, 3, pic.size()[1], pic.size()[2])
goal = goal.expand(1, 3, goal.size()[1], goal.size()[2])

def createIdentityGrid(w, h):
    grid = torch.zeros(1, w, h, 2);
    for x in range(w):
        for y in range(h):
            grid[0][x][y][1] = 2 / w * (0.5 + x) - 1
            grid[0][x][y][0] = 2 / h * (0.5 + y) - 1
    return grid

w = 256; h=256 #hardcoded imagesize

grid = createIdentityGrid(w, h)
grid.requires_grad = True

for i in range(300):
    goal_pred = torch.nn.functional.grid_sample(pic, grid)[0]
    goal_pred = (empty + 0.75 * goal_pred).clamp(min=0, max=1)
    out = goal_pred

    loss = (goal_pred - goal).pow(2).sum()
    loss.backward()

    with torch.no_grad():
        grid -= grid.grad * (1e-2)
        grid.grad.zero_()

FromTensor(out[0]).show()

these are the result:

It is actualy working with this this easy example, but I observe some weird behaviour. The grid just starts to change on one side. Why is this so, and why doesn't the complete grid change instantly? Is there some obvious part I am missing?

prosti · Answer 1 · 2019-07-05T15:53:52.253

from PIL import Image
import torch

ToTensor = torchvision.transforms.ToTensor()
FromTensor = torchvision.transforms.ToPILImage()

lr = 1
backround= ToTensor(Image.open(r"C:\Users\dj\Pictures\Saved Pictures\background.png"))
pic = ToTensor(Image.open(r"C:\Users\dj\Pictures\Saved Pictures\input.png"))
goal = ToTensor(Image.open(r"C:\Users\dj\Pictures\Saved Pictures\result.png"))

empty = backround.expand(1, 3, backround.size()[1], backround.size()[2])
pic = pic.expand(1, 3, pic.size()[1], pic.size()[2])
goal = goal.expand(1, 3, goal.size()[1], goal.size()[2])

def createIdentityGrid(w, h):
    grid = torch.zeros(1, w, h, 2);
    for x in range(w):
        for y in range(h):
            grid[0][x][y][1] = 2 / w * (0.5 + y) - 1
            grid[0][x][y][0] = 2 / h * (0.5 + x) - 1
    return grid

w = 256; h=256 #hardcoded imagesize

grid = createIdentityGrid(w, h)
grid.requires_grad = True

for i in range(9):
    goal_pred = torch.nn.functional.grid_sample(pic, grid, mode="bilinear")[0]
    goal_pred = F.relu(empty + 0.75 * goal_pred)
    out = goal_pred

    loss = (goal_pred - goal).pow(2).sum()
    loss.backward()

    with torch.no_grad():
        grid -= grid.grad * lr
        lr = lr/1.1 #learning rate a0-ing
        grid.grad.zero_()

FromTensor(out[0]).show()

It is actualy working with this this easy example, but I observe some weird behaviour. The grid just starts to change on one side. Why is this so, and why doesn't the complete grid change instantly?

I don't know. I just duped your example, and for me it goes from bottom up.

PyTorch Image transformation with gridsampler, weird behaviour while optimizing grid

1 Answers1