I write iOS plug in to integrate MetalFX Spatial Upscaling to Unity URP project.
C# Code in Unity:
namespace UnityEngine.Rendering.Universal
{
///
/// Renders the post-processing effect stack.
///
internal class PostProcessPass : ScriptableRenderPass
{
RenderTexture _dstRT = null;
[DllImport ("__Internal")]
private static extern void MetalFX_SpatialScaling (IntPtr srcTexture, IntPtr dstTexture, IntPtr outTexture);
}
}
void RenderFinalPass(CommandBuffer cmd, ref RenderingData renderingData)
{
// ......
case ImageUpscalingFilter.MetalFX:
{
var upscaleRtDesc = tempRtDesc;
upscaleRtDesc.width = cameraData.pixelWidth;
upscaleRtDesc.height = cameraData.pixelHeight;
RenderingUtils.ReAllocateIfNeeded(ref m_UpscaledTarget, upscaleRtDesc, FilterMode.Point, TextureWrapMode.Clamp, name: "_UpscaledTexture");
var metalfxInputSize = new Vector2(cameraData.cameraTargetDescriptor.width, cameraData.cameraTargetDescriptor.height);
if (_dstRT == null)
{
_dstRT = new RenderTexture(upscaleRtDesc.width, upscaleRtDesc.height, 0, RenderTextureFormat.ARGB32);
_dstRT.Create();
}
// call native plugin
cmd.SetRenderTarget(m_UpscaledTarget, RenderBufferLoadAction.DontCare, RenderBufferStoreAction.Store, RenderBufferLoadAction.DontCare, RenderBufferStoreAction.DontCare);
MetalFX_SpatialScaling(sourceTex.rt.GetNativeTexturePtr(), m_UpscaledTarget.rt.GetNativeTexturePtr(), _dstRT.GetNativeTexturePtr());
Graphics.CopyTexture(_dstRT, m_UpscaledTarget.rt);
sourceTex = m_UpscaledTarget;
PostProcessUtils.SetSourceSize(cmd, upscaleRtDesc);
break;
}
// .....
}
Objective-c Code in iOS:
head file:
#import <Foundation/Foundation.h>
#import <MetalFX/MTLFXSpatialScaler.h>
@protocol MTLTexture;
@protocol MTLDevice;
API_AVAILABLE(ios(16.0))
@interface MetalFXDelegate : NSObject
{
int mode;
id _device;
id _commandQueue;
id _outTexture;
id _mfxSpatialScaler;
id _mfxSpatialEncoder;
};
(void)SpatialScaling: (MTLTextureRef) srcTexture
dstTexure: (MTLTextureRef) dstTexture
outTexure: (MTLTextureRef) outTexture;
(void)saveTexturePNG: (MTLTextureRef) texture
url: (CFURLRef) url;
@end
m file:
#import "MetalFXOC.h"
@implementation MetalFXDelegate
(id)init
{
self = [super init];
return self;
}
static MetalFXDelegate* delegateObject = nil;
(void)SpatialScaling: (MTLTextureRef) srcTexture
dstTexture: (MTLTextureRef) dstTexture
outTexture: (MTLTextureRef) outTexture {
int width = (int)srcTexture.width;
int height = (int)srcTexture.height;
int dstWidth = (int)dstTexture.width;
int dstHeight = (int)dstTexture.height;
if (_mfxSpatialScaler == nil) {
MTLFXSpatialScalerDescriptor* desc;
desc = [[MTLFXSpatialScalerDescriptor alloc]init];
desc.inputWidth = width;
desc.inputHeight = height;
desc.outputWidth = dstWidth; ///_screenWidth
desc.outputHeight = dstHeight; ///_screenHeight
desc.colorTextureFormat = srcTexture.pixelFormat;
desc.outputTextureFormat = dstTexture.pixelFormat;
if (@available(iOS 16.0, *)) {
desc.colorProcessingMode = MTLFXSpatialScalerColorProcessingModePerceptual;
} else {
// Fallback on earlier versions
}
_device = MTLCreateSystemDefaultDevice();
_mfxSpatialScaler = [desc newSpatialScalerWithDevice:_device];
if (_mfxSpatialScaler == nil) {
return;
}
_commandQueue = [_device newCommandQueue];
MTLTextureDescriptor *texdesc = [[MTLTextureDescriptor alloc] init];
texdesc.width = (int)dstTexture.width;
texdesc.height = (int)dstTexture.height;
texdesc.storageMode = MTLStorageModePrivate;
texdesc.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite;
texdesc.pixelFormat = dstTexture.pixelFormat;
_outTexture = [_device newTextureWithDescriptor:texdesc];
}
id upscaleCommandBuffer = [_commandQueue commandBuffer];
upscaleCommandBuffer.label = @"Upscale Command Buffer";
_mfxSpatialScaler.colorTexture = srcTexture;
_mfxSpatialScaler.outputTexture = _outTexture;
[_mfxSpatialScaler encodeToCommandBuffer:upscaleCommandBuffer];
// outTexture = _outTexture;
id textureCommandBuffer = [_commandQueue commandBuffer];
id _mfxSpatialEncoder =[textureCommandBuffer blitCommandEncoder];
[_mfxSpatialEncoder copyFromTexture:_outTexture toTexture:outTexture];
[_mfxSpatialEncoder endEncoding];
[upscaleCommandBuffer commit];
}
@end
extern "C" {
void MetalFX_SpatialScaling(void* srcTexturePtr, void* dstTexturePtr, void* outTexturePtr) {
if (delegateObject == nil) {
if (@available(iOS 16.0, *)) {
delegateObject = [[MetalFXDelegate alloc] init];
} else {
// Fallback on earlier versions
}
}
if (srcTexturePtr == nil || dstTexturePtr == nil || outTexturePtr == nil) {
return;
}
id<MTLTexture> srcTexture = (__bridge id<MTLTexture>)(void *)srcTexturePtr;
id<MTLTexture> dstTexture = (__bridge id<MTLTexture>)(void *)dstTexturePtr;
id<MTLTexture> outTexture = (__bridge id<MTLTexture>)(void *)outTexturePtr;
if (@available(iOS 16.0, *)) {
[delegateObject SpatialScaling: srcTexture
dstTexture: dstTexture
outTexture: outTexture];
} else {
// Fallback on earlier versions
}
return;
}
}
With the C# and objective code, the appear on screen is black.
If I save the MTLTexture to PNG in ios plug in, the PNG is ok(not black), so I think the outTexture: outTexture write to unity is failed.