Hi am trying to create some feedback loops in metal for an app I am working on. However I am getting some strange artifacts. I suspect this is due to something in my pipeline, or with how I am sampling the previous frame. In the code there are extra pipelines set up for a second buffer, as eventually I will need that to make a turing pattern. But right now I have reduced the code as much as possible to isolate the issue.
Right now it is just drawing a small circle in the top left corner. Once that is drawn the buffer A fragment samples itself with a one pixel displacement. This causes the circle to stretch out down the screen with every loop. The issue is that I am getting coloured lines appearing ahead of the stretching circle. This same artifact appears when I try and do recursive blurs or other types of feedback loops. The Main buffer is only there to mix the two buffers at the end of the process. I think it can be ignored for troubshooting this. Anyone run into this before?
Video here: https://photos.app.goo.gl/ThRJHddo2xKke6CN7
Screen shot here: Picture of artifact
Calls to metal controller in my viewController
'''
override func viewDidLoad() {
super.viewDidLoad()
metal.initMetalLayer(mView: metalView) // make metal layer
metal.initMetalPipeline()//make metal pipeline
metal.commandQueue = metal.device.makeCommandQueue()//make metal command Queue
timer = CADisplayLink(target: self, selector: #selector(gameloop)) //make the timer to trigger the render call
timer.add(to: RunLoop.main, forMode: .default) //ties the screen refresh to the render call
}
@objc func gameloop() {
autoreleasepool {
metal.currentTime = Float(CACurrentMediaTime() - metal.startTime)
//print(metal.currentTime)
// metal.bufferBRender()
metal.bufferARender()
metal.mainRender() //TURNS METAL RENDER OFF OR ON
}
}
'''
Metal declarations
'''
//
// MTL.swift
// Habituate
//
// Created by Brendan Tipney on 2021-01-14.
//
import Foundation
import Metal
import UIKit
import MetalKit
struct MTL {
var device: MTLDevice!
var metalLayer: CAMetalLayer!
let vertexData: [Float] = [
-1.0, 1.0, 0.0,
1.0, 1.0, 0.0,
-1.0, -1.0, 0.0,
1.0,-1.0,0.0
]
var vertexBuffer: MTLBuffer!
var textureA: MTLTexture!
var textureB: MTLTexture!
//MARK: - AUX data
var layoutData = simd_float4(390,727,0.75,1.0)
var habitData = [vector_float4(0.25,1.0,0.75,1.0),vector_float4(0.1,1.0,0.3,0.4)]
var touchData = vector_float4(0,0,0,0)
var startTime = CACurrentMediaTime()
var currentTime = Float(CACurrentMediaTime())
//MARK: - RCTDIFData
var RCTDIFData = vector_float4(0.055,0.064,2.0,1.0)
var pipelineState: MTLRenderPipelineState! //DO I need more then one pipeline state?
var pipelineStateB: MTLRenderPipelineState!
var pipelineStateA: MTLRenderPipelineState!
var commandQueue: MTLCommandQueue!
var timer: CADisplayLink!
mutating func initMetalLayer(mView : MTKView) {
device = MTLCreateSystemDefaultDevice()
metalLayer = CAMetalLayer()
metalLayer.device = device
metalLayer.pixelFormat = .bgra8Unorm
metalLayer.framebufferOnly = false// Set to false as I intend to read from the frame buffer for feedback loops. Set to true if not needed
metalLayer.frame = mView.layer.frame
mView.layer.addSublayer(metalLayer)
let viewSize = simd_float2(Float(mView.layer.frame.size.width), Float(mView.layer.frame.size.height))
layoutData = simd_float4(viewSize.x, viewSize.y, 0.75, 1.0) //Trying to load the size data.
print("view = \(viewSize)")
let dataSize = vertexData.count * MemoryLayout.size(ofValue: vertexData[0])
vertexBuffer = device.makeBuffer(bytes: vertexData, length: dataSize, options: []) //makes the buffer with the vertex data.
}
mutating func initMetalPipeline(){
let defaultLibrary = device.makeDefaultLibrary()!
let vertexProgram = defaultLibrary.makeFunction(name: "vertex_main")
let fragmentProgram = defaultLibrary.makeFunction(name: "fragment_main")
let fragmentProgramBuffA = defaultLibrary.makeFunction(name: "fragment_BuffA")
let fragmentProgramBuffB = defaultLibrary.makeFunction(name: "fragment_BuffB")
//MARK: - VertexDescriptor
let vertexDescriptor = MTLVertexDescriptor()//I think this is where the buffers are stored
vertexDescriptor.attributes[0].format = .float3
vertexDescriptor.attributes[0].bufferIndex = 0
vertexDescriptor.attributes[0].offset = 0
vertexDescriptor.layouts[0].stride = MemoryLayout<float3>.stride
//MARK: - Texture Descriptor
let textureDescriptor = MTLTextureDescriptor() //None of this textureDescriptors may be needed
textureDescriptor.textureType = MTLTextureType.type2D
textureDescriptor.width = Int(layoutData.x)
textureDescriptor.height = Int(layoutData.y)
textureDescriptor.pixelFormat = MTLPixelFormat.bgra8Unorm
textureDescriptor.usage = [.renderTarget, .shaderRead]
textureA = device.makeTexture(descriptor: textureDescriptor)
textureB = device.makeTexture(descriptor: textureDescriptor)
//MARK: - Pipeline Descriptor
let pipelineStateDescriptor = MTLRenderPipelineDescriptor()
pipelineStateDescriptor.vertexFunction = vertexProgram
pipelineStateDescriptor.fragmentFunction = fragmentProgram
pipelineStateDescriptor.colorAttachments[0].pixelFormat = .bgra8Unorm
pipelineStateDescriptor.vertexDescriptor = vertexDescriptor
pipelineState = try! device.makeRenderPipelineState(descriptor: pipelineStateDescriptor)
//MARK: - BufferA Pipeline Descriptor
pipelineStateDescriptor.fragmentFunction = fragmentProgramBuffA
pipelineStateA = try! device.makeRenderPipelineState(descriptor: pipelineStateDescriptor)
//MARK: - BufferB Pipeline Descriptor
pipelineStateDescriptor.fragmentFunction = fragmentProgramBuffB
pipelineStateB = try! device.makeRenderPipelineState(descriptor: pipelineStateDescriptor)
}
//MARK: - Main Render
mutating func mainRender() {
let commandBuffer = commandQueue.makeCommandBuffer()!
guard let drawable = metalLayer?.nextDrawable() else { return }
let renderPassDescriptor = MTLRenderPassDescriptor()
renderPassDescriptor.colorAttachments[0].texture = drawable.texture
renderPassDescriptor.colorAttachments[0].loadAction = .clear //PERHAPS SETTING THIS TO LOAD CAN MAKE A FB LOOP
renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColor(red: 0.0, green: 1.0, blue: 0.0, alpha: 0.0)
let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor)!
renderEncoder.label = "Main"
renderEncoder.setRenderPipelineState(pipelineState)
renderEncoder.setVertexBuffer(vertexBuffer, offset: 0, index: 0)//this is where the vertex buffer is loaded
renderEncoder.setFragmentTexture(textureA, index: 0)
renderEncoder.setFragmentTexture(textureB, index: 1)
renderEncoder.setFragmentBytes(&layoutData, length: MemoryLayout.size(ofValue: layoutData), index: 1) //this sends the dimentions to the shader
renderEncoder.setFragmentBytes(&habitData, length: 4 * MemoryLayout.size(ofValue: habitData), index: 2)
renderEncoder.setFragmentBytes(&touchData, length: MemoryLayout.size(ofValue: touchData), index: 3)
renderEncoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4, instanceCount: 1)
renderEncoder.endEncoding()
commandBuffer.present(drawable)
commandBuffer.commit()
}
//MARK: - Buffer B Render
mutating func bufferARender(){
let commandBuffer = commandQueue.makeCommandBuffer()!
let renderPassDescriptorA = MTLRenderPassDescriptor()
renderPassDescriptorA.colorAttachments[0].texture = textureA //set back to this after testing
renderPassDescriptorA.colorAttachments[0].loadAction = .load
renderPassDescriptorA.colorAttachments[0].storeAction = .store
renderPassDescriptorA.colorAttachments[0].clearColor = MTLClearColor(red: 0.0, green: 1.0, blue: 0.0, alpha: 1.0)
let renderEncoderA = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptorA)!
renderEncoderA.label = "BufferA"
renderEncoderA.setRenderPipelineState(pipelineStateA)
renderEncoderA.setVertexBuffer(vertexBuffer, offset: 0, index: 0)//this is where the vertex buffer is loaded
renderEncoderA.setFragmentTexture(textureA, index: 0)
renderEncoderA.setFragmentTexture(textureB, index: 1)
renderEncoderA.setFragmentBytes(&layoutData, length: MemoryLayout.size(ofValue: layoutData), index: 1) //this sends the dimentions to the shader
renderEncoderA.setFragmentBytes(&habitData, length: 4 * MemoryLayout.size(ofValue: habitData), index: 2)
renderEncoderA.setFragmentBytes(&touchData, length: MemoryLayout.size(ofValue: touchData), index: 3)
renderEncoderA.setFragmentBytes(&RCTDIFData, length: MemoryLayout.size(ofValue: RCTDIFData), index: 4)
renderEncoderA.setFragmentBytes(¤tTime, length: MemoryLayout.size(ofValue: currentTime), index: 5)
renderEncoderA.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4, instanceCount: 1)
renderEncoderA.endEncoding()
//commandBuffer.present(drawable)
commandBuffer.commit()
}
mutating func bufferBRender(){
let commandBuffer = commandQueue.makeCommandBuffer()!
let renderPassDescriptorB = MTLRenderPassDescriptor()
//renderPassDescriptorB.colorAttachments[0].texture = drawable.texture
renderPassDescriptorB.colorAttachments[0].texture = textureB //set back to this after testing
renderPassDescriptorB.colorAttachments[0].loadAction = .load
renderPassDescriptorB.colorAttachments[0].storeAction = .store
renderPassDescriptorB.colorAttachments[0].clearColor = MTLClearColor(red: 0.0, green: 0.0, blue: 1.0, alpha: 1.0)
let renderEncoderB = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptorB)!
renderEncoderB.label = "BufferB"
renderEncoderB.setRenderPipelineState(pipelineStateB)
renderEncoderB.setVertexBuffer(vertexBuffer, offset: 0, index: 0)//this is where the vertex buffer is loaded
renderEncoderB.setFragmentTexture(textureA, index: 0)
renderEncoderB.setFragmentTexture(textureB, index: 1)
renderEncoderB.setFragmentBytes(&layoutData, length: MemoryLayout.size(ofValue: layoutData), index: 1) //this sends the dimentions to the shader
renderEncoderB.setFragmentBytes(&habitData, length: 4 * MemoryLayout.size(ofValue: habitData), index: 2)
renderEncoderB.setFragmentBytes(&touchData, length: MemoryLayout.size(ofValue: touchData), index: 3)
renderEncoderB.setFragmentBytes(&RCTDIFData, length: MemoryLayout.size(ofValue: RCTDIFData), index: 4)
renderEncoderB.setFragmentBytes(¤tTime, length: MemoryLayout.size(ofValue: currentTime), index: 5)
renderEncoderB.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4, instanceCount: 1)
renderEncoderB.endEncoding()
//commandBuffer.present(drawable)
commandBuffer.commit()
}
mutating func getDeviceDetails(){
let window = UIApplication.shared.windows[0]
let safeViewTop = window.safeAreaInsets.top
}
mutating func getHabitDetails(_ habits: HabitData){
}
}
'''
Vertex and Fragment shaders
'''
#include <metal_stdlib>
using namespace metal;
struct VertexIn {
float3 position [[attribute(0)]];
};
struct VertexOut {
float4 position [[position]];
};
vertex VertexOut vertex_main(VertexIn in [[stage_in]],
const device packed_float3* vertex_array [[buffer(0)]],
unsigned int vid [[ vertex_id ]]) {
VertexOut out;
out.position = float4(vertex_array[vid].xy,1.0);
//out.data = float4(in.data, 1);
return out;
}
fragment float4 fragment_main( VertexOut in [[stage_in]],
constant float4 &layoutData [[ buffer(1) ]],
constant array<float4, 2> &habitData [[ buffer(2) ]],
constant float4 &touchData [[ buffer(3) ]],
texture2d<float, access::sample> BuffA [[ texture(0) ]],
texture2d<float, access::sample> BuffB [[ texture(1) ]]) {
float4 fragIn = in.position;
float2 uv = float2((fragIn.x/layoutData.x), (fragIn.y/layoutData.y));
float4 fragColor = float4(0.0,0.0,0.0,1.0);
constexpr sampler s(coord::normalized, filter::bicubic, address::clamp_to_edge,compare_func:: less);
float4 texA = BuffA.sample(s, uv);
fragColor = texA;
fragColor.w = 1.0;
return fragColor;
}
fragment float4 fragment_BuffA( VertexOut in [[stage_in]],
constant float4 &layoutData [[ buffer(1) ]],
constant array<float4, 2> &habitData [[ buffer(2) ]],
constant float4 &touchData [[ buffer(3) ]],
constant float4 &RCTDIFData [[ buffer(4) ]],
constant float ¤tTime [[ buffer(5)]],
texture2d<float, access::sample> BuffA [[ texture(0) ]],
texture2d<float, access::sample> BuffB [[ texture(1) ]]) {
float4 fragIn = in.position;
float2 uv = float2((fragIn.x/layoutData.x), (fragIn.y/layoutData.y));//Need to load the screen size here or it will only look right on my phone.
float2 pxlRatio = float2(1)/layoutData.xy;
float4 fragColor = float4(0.0,0.0,0.0,1.0);
constexpr sampler s(coord::normalized, filter::nearest, address::clamp_to_zero);
float cornerCircle = step(distance(float2(0,0), uv),0.1);
float4 texA = BuffA.sample(s, uv+float2(-1,-1)*pxlRatio);
fragColor = float4(cornerCircle);
fragColor += texA;
fragColor.w = 1.;
return fragColor;
}
'''
