Skip to content

Commit d913373

Browse files
zhangqianzezhangqianze
committed
arkit done
1 parent 9d8a5f5 commit d913373

10 files changed

Lines changed: 711 additions & 2 deletions

File tree

‎iOS/APIExample.xcodeproj/project.pbxproj‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,11 @@
6161
033A9F89252D8B6C00BC26E1 /* PrecallTest.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 033A9F8B252D8B6C00BC26E1 /* PrecallTest.storyboard */; };
6262
033A9F8E252D8FF300BC26E1 /* JoinMultiChannel.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 033A9F45252D89C800BC26E1 /* JoinMultiChannel.storyboard */; };
6363
034C625E2524A06800296ECF /* VoiceChanger.swift in Sources */ = {isa = PBXBuildFile; fileRef = 034C625D2524A06800296ECF /* VoiceChanger.swift */; };
64+
0364C1FC2551AD6D00C6C0AE /* ARKit.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0364C1F82551AD6D00C6C0AE /* ARKit.swift */; };
65+
0364C1FD2551AD6D00C6C0AE /* ARKit.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 0364C1F92551AD6D00C6C0AE /* ARKit.storyboard */; };
66+
0364C2022551B19800C6C0AE /* ARVideoSource.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0364C2002551B19800C6C0AE /* ARVideoSource.swift */; };
67+
0364C2032551B19800C6C0AE /* ARVideoRenderer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0364C2012551B19800C6C0AE /* ARVideoRenderer.swift */; };
68+
0364C2052551B46100C6C0AE /* AR.scnassets in Resources */ = {isa = PBXBuildFile; fileRef = 0364C2042551B46100C6C0AE /* AR.scnassets */; };
6469
036C42AC24D292A700A59000 /* AgoraCameraSourceMediaIO.swift in Sources */ = {isa = PBXBuildFile; fileRef = 036C42AB24D292A700A59000 /* AgoraCameraSourceMediaIO.swift */; };
6570
036C42B024D2955D00A59000 /* AgoraCameraSourcePush.swift in Sources */ = {isa = PBXBuildFile; fileRef = 036C42AF24D2955D00A59000 /* AgoraCameraSourcePush.swift */; };
6671
036C42B524D2A3C600A59000 /* AgoraMetalRender.swift in Sources */ = {isa = PBXBuildFile; fileRef = 036C42B324D2A3C600A59000 /* AgoraMetalRender.swift */; };
@@ -257,7 +262,13 @@
257262
033A9F88252D8B6700BC26E1 /* zh-Hans */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = "zh-Hans"; path = "zh-Hans.lproj/StreamEncryption.strings"; sourceTree = "<group>"; };
258263
033A9F8A252D8B6C00BC26E1 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/PrecallTest.storyboard; sourceTree = "<group>"; };
259264
033A9F8D252D8B7000BC26E1 /* zh-Hans */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = "zh-Hans"; path = "zh-Hans.lproj/PrecallTest.strings"; sourceTree = "<group>"; };
265+
03414B502551C98E00AB114D /* zh-Hans */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = "zh-Hans"; path = "zh-Hans.lproj/ARKit.strings"; sourceTree = "<group>"; };
260266
034C625D2524A06800296ECF /* VoiceChanger.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = VoiceChanger.swift; sourceTree = "<group>"; };
267+
0364C1F82551AD6D00C6C0AE /* ARKit.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ARKit.swift; sourceTree = "<group>"; };
268+
0364C1FA2551AD6D00C6C0AE /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/ARKit.storyboard; sourceTree = "<group>"; };
269+
0364C2002551B19800C6C0AE /* ARVideoSource.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ARVideoSource.swift; sourceTree = "<group>"; };
270+
0364C2012551B19800C6C0AE /* ARVideoRenderer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ARVideoRenderer.swift; sourceTree = "<group>"; };
271+
0364C2042551B46100C6C0AE /* AR.scnassets */ = {isa = PBXFileReference; lastKnownFileType = wrapper.scnassets; path = AR.scnassets; sourceTree = "<group>"; };
261272
036C42AB24D292A700A59000 /* AgoraCameraSourceMediaIO.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AgoraCameraSourceMediaIO.swift; sourceTree = "<group>"; };
262273
036C42AF24D2955D00A59000 /* AgoraCameraSourcePush.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AgoraCameraSourcePush.swift; sourceTree = "<group>"; };
263274
036C42B324D2A3C600A59000 /* AgoraMetalRender.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AgoraMetalRender.swift; sourceTree = "<group>"; };
@@ -575,6 +586,24 @@
575586
path = VoiceChanger;
576587
sourceTree = "<group>";
577588
};
589+
0364C1F52551AD6D00C6C0AE /* ARKit */ = {
590+
isa = PBXGroup;
591+
children = (
592+
0364C1F82551AD6D00C6C0AE /* ARKit.swift */,
593+
0364C1F92551AD6D00C6C0AE /* ARKit.storyboard */,
594+
);
595+
path = ARKit;
596+
sourceTree = "<group>";
597+
};
598+
0364C1FF2551B19800C6C0AE /* ARKit */ = {
599+
isa = PBXGroup;
600+
children = (
601+
0364C2002551B19800C6C0AE /* ARVideoSource.swift */,
602+
0364C2012551B19800C6C0AE /* ARVideoRenderer.swift */,
603+
);
604+
path = ARKit;
605+
sourceTree = "<group>";
606+
};
578607
036C42AA24D292A700A59000 /* ExternalVideo */ = {
579608
isa = PBXGroup;
580609
children = (
@@ -738,6 +767,7 @@
738767
03D13BFF24488F1E00B599B3 /* Common */ = {
739768
isa = PBXGroup;
740769
children = (
770+
0364C1FF2551B19800C6C0AE /* ARKit */,
741771
0339BE73251EF728007D4FDD /* RtcChannelPublishPlugin */,
742772
036CBA4025198E9100D74FAD /* CustomEncryption */,
743773
036C42B924D5853200A59000 /* RawDataApi */,
@@ -789,6 +819,7 @@
789819
A75A56D724A0603000D0089E /* Advanced */ = {
790820
isa = PBXGroup;
791821
children = (
822+
0364C1F52551AD6D00C6C0AE /* ARKit */,
792823
033A9F01252D61FB00BC26E1 /* RawMediaData */,
793824
033A9F03252D61FB00BC26E1 /* RTMPInjection */,
794825
033A9F05252D61FB00BC26E1 /* RTMPStreaming */,
@@ -820,6 +851,7 @@
820851
03D13BD82448758B00B599B3 /* Assets.xcassets */,
821852
03D13BDA2448758B00B599B3 /* LaunchScreen.storyboard */,
822853
033A9F2C252D737900BC26E1 /* Localizable.strings */,
854+
0364C2042551B46100C6C0AE /* AR.scnassets */,
823855
);
824856
name = "Supporting Files";
825857
sourceTree = "<group>";
@@ -961,7 +993,9 @@
961993
03C1647325447249002C6F4E /* model.bundle in Resources */,
962994
033A9F4D252D89DB00BC26E1 /* CustomAudioSource.storyboard in Resources */,
963995
033A9F84252D8B6400BC26E1 /* StreamEncryption.storyboard in Resources */,
996+
0364C2052551B46100C6C0AE /* AR.scnassets in Resources */,
964997
033A9F30252D860100BC26E1 /* JoinChannelAudio.storyboard in Resources */,
998+
0364C1FD2551AD6D00C6C0AE /* ARKit.storyboard in Resources */,
965999
033A9F75252D8B4800BC26E1 /* ScreenShare.storyboard in Resources */,
9661000
033A9F5C252D89FD00BC26E1 /* CustomVideoSourcePush.storyboard in Resources */,
9671001
03D13BD92448758B00B599B3 /* Assets.xcassets in Resources */,
@@ -1070,6 +1104,7 @@
10701104
033A9EEA252D5F5E00BC26E1 /* JoinMultiChannel.swift in Sources */,
10711105
0339BE64251DCA3B007D4FDD /* GlobalSettings.swift in Sources */,
10721106
0339BE85251EF728007D4FDD /* AgoraMediaPlayerEx.cpp in Sources */,
1107+
0364C1FC2551AD6D00C6C0AE /* ARKit.swift in Sources */,
10731108
0339BE89251EF728007D4FDD /* AudioFrameObserver.cpp in Sources */,
10741109
8407E0942472320800AC5DE8 /* (null) in Sources */,
10751110
036C42B524D2A3C600A59000 /* AgoraMetalRender.swift in Sources */,
@@ -1100,8 +1135,10 @@
11001135
033A9F08252D61FC00BC26E1 /* RTMPInjection.swift in Sources */,
11011136
A7847F922458062900469187 /* StatisticsInfo.swift in Sources */,
11021137
0339BE72251EF075007D4FDD /* MediaPlayer.swift in Sources */,
1138+
0364C2032551B19800C6C0AE /* ARVideoRenderer.swift in Sources */,
11031139
0385767E2521E5A0003C369A /* MediaChannelRelay.swift in Sources */,
11041140
03BCEC762449EB5000ED7177 /* LogViewController.swift in Sources */,
1141+
0364C2022551B19800C6C0AE /* ARVideoSource.swift in Sources */,
11051142
0339D6D624E91CEB008739CD /* QuickSwitchChannel.swift in Sources */,
11061143
A7847F942458089E00469187 /* AgoraExtension.swift in Sources */,
11071144
03BCEC50244938C500ED7177 /* BaseViewController.swift in Sources */,
@@ -1307,6 +1344,15 @@
13071344
name = PrecallTest.storyboard;
13081345
sourceTree = "<group>";
13091346
};
1347+
0364C1F92551AD6D00C6C0AE /* ARKit.storyboard */ = {
1348+
isa = PBXVariantGroup;
1349+
children = (
1350+
0364C1FA2551AD6D00C6C0AE /* Base */,
1351+
03414B502551C98E00AB114D /* zh-Hans */,
1352+
);
1353+
name = ARKit.storyboard;
1354+
sourceTree = "<group>";
1355+
};
13101356
03D13BD52448758900B599B3 /* Main.storyboard */ = {
13111357
isa = PBXVariantGroup;
13121358
children = (

‎iOS/APIExample/AR.scnassets/displayer.scn‎

18 KB
Binary file not shown.

‎iOS/APIExample/Common/ARKit/ARVideoRenderer.swift‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
//
2+
// ARVideoRenderer.swift
3+
// Agora-Video-With-ARKit
4+
//
5+
// Created by GongYuhua on 2017/12/27.
6+
// Copyright © 2017年 Agora.io All rights reserved.
7+
//
8+
9+
import Foundation
10+
import MetalKit
11+
import SceneKit
12+
import AgoraRtcKit
13+
14+
class ARVideoRenderer : NSObject {
15+
fileprivate var yTexture: MTLTexture?
16+
fileprivate var uTexture: MTLTexture?
17+
fileprivate var vTexture: MTLTexture?
18+
fileprivate var rgbTexture: MTLTexture?
19+
20+
fileprivate let device = MTLCreateSystemDefaultDevice()
21+
fileprivate var commandQueue: MTLCommandQueue?
22+
23+
fileprivate var defaultLibrary: MTLLibrary?
24+
25+
fileprivate var threadsPerThreadgroup = MTLSizeMake(16, 16, 1)
26+
fileprivate var threadgroupsPerGrid = MTLSizeMake(128, 96, 1)
27+
fileprivate var pipelineState: MTLComputePipelineState?
28+
29+
var renderNode: SCNNode?
30+
}
31+
32+
extension ARVideoRenderer: AgoraVideoSinkProtocol {
33+
func shouldInitialize() -> Bool {
34+
defaultLibrary = device?.makeDefaultLibrary()
35+
36+
if let device = device, let function = defaultLibrary?.makeFunction(name: "writeRGBFromYUV") {
37+
pipelineState = try? device.makeComputePipelineState(function: function)
38+
}
39+
40+
commandQueue = device?.makeCommandQueue()
41+
42+
return true
43+
}
44+
45+
func shouldStart() {
46+
47+
}
48+
49+
func shouldStop() {
50+
51+
}
52+
53+
func shouldDispose() {
54+
yTexture = nil
55+
uTexture = nil
56+
vTexture = nil
57+
rgbTexture = nil
58+
59+
renderNode?.geometry?.firstMaterial?.diffuse.contents = createEmptyRGBTexture(width: 1, height: 1)
60+
}
61+
62+
func bufferType() -> AgoraVideoBufferType {
63+
return .rawData
64+
}
65+
66+
func pixelFormat() -> AgoraVideoPixelFormat {
67+
return .I420
68+
}
69+
70+
func renderRawData(_ rawData: UnsafeMutableRawPointer, size: CGSize, rotation: AgoraVideoRotation) {
71+
guard let node = renderNode else {
72+
return
73+
}
74+
75+
let width = Int(size.width)
76+
let height = Int(size.height)
77+
78+
yTexture = createTexture(withData: rawData,
79+
width: width,
80+
height: height)
81+
uTexture = createTexture(withData: rawData + width * height,
82+
width: width / 2,
83+
height: height / 2)
84+
vTexture = createTexture(withData: rawData + width * height * 5 / 4,
85+
width: width / 2,
86+
height: height / 2)
87+
88+
rgbTexture = createEmptyRGBTexture(width: width, height: height)
89+
90+
node.geometry?.firstMaterial?.diffuse.contents = rgbTexture
91+
renderRGBTexture()
92+
}
93+
}
94+
95+
private extension ARVideoRenderer {
96+
func createTexture(withData data: UnsafeMutableRawPointer, width: Int, height: Int) -> MTLTexture? {
97+
let descriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .r8Uint,
98+
width: width,
99+
height: height,
100+
mipmapped: false)
101+
let texture = device?.makeTexture(descriptor: descriptor)
102+
texture?.replace(region: MTLRegionMake2D(0, 0, width, height),
103+
mipmapLevel: 0,
104+
withBytes: data,
105+
bytesPerRow: width)
106+
107+
return texture
108+
}
109+
110+
func createEmptyRGBTexture(width: Int, height: Int) -> MTLTexture? {
111+
let rgbaDescriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .rgba16Float,
112+
width: width,
113+
height: height,
114+
mipmapped: false)
115+
rgbaDescriptor.usage = [.shaderWrite, .shaderRead]
116+
let rgbTexture = device?.makeTexture(descriptor: rgbaDescriptor)
117+
return rgbTexture
118+
}
119+
120+
func renderRGBTexture() {
121+
guard let state = pipelineState,
122+
let buffer = commandQueue?.makeCommandBuffer(),
123+
let encoder = buffer.makeComputeCommandEncoder() else {
124+
return
125+
}
126+
127+
encoder.setComputePipelineState(state)
128+
encoder.setTexture(yTexture, index: 0)
129+
encoder.setTexture(uTexture, index: 1)
130+
encoder.setTexture(vTexture, index: 2)
131+
encoder.setTexture(rgbTexture, index: 3)
132+
encoder.dispatchThreadgroups(threadgroupsPerGrid,
133+
threadsPerThreadgroup: threadsPerThreadgroup)
134+
encoder.endEncoding()
135+
136+
buffer.commit()
137+
}
138+
}

‎iOS/APIExample/Common/ARKit/ARVideoSource.swift‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
//
2+
// ARVideoSource.swift
3+
// Agora-Video-With-ARKit
4+
//
5+
// Created by GongYuhua on 2018/1/11.
6+
// Copyright © 2018年 Agora. All rights reserved.
7+
//
8+
9+
import UIKit
10+
import AgoraRtcKit
11+
12+
class ARVideoSource: NSObject, AgoraVideoSourceProtocol {
13+
var consumer: AgoraVideoFrameConsumer?
14+
15+
func shouldInitialize() -> Bool { return true }
16+
17+
func shouldStart() { }
18+
19+
func shouldStop() { }
20+
21+
func shouldDispose() { }
22+
23+
func bufferType() -> AgoraVideoBufferType {
24+
return .pixelBuffer
25+
}
26+
27+
func contentHint() -> AgoraVideoContentHint {
28+
return .none
29+
}
30+
31+
func captureType() -> AgoraVideoCaptureType {
32+
return .camera
33+
}
34+
35+
func sendBuffer(_ buffer: CVPixelBuffer, timestamp: TimeInterval) {
36+
let time = CMTime(seconds: timestamp, preferredTimescale: 1000)
37+
consumer?.consumePixelBuffer(buffer, withTimestamp: time, rotation: .rotation90)
38+
}
39+
}

‎iOS/APIExample/Common/ExternalVideo/AgoraMetalShader.metal‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,3 +47,24 @@ fragment float4 displayNV12Texture(TextureMappingVertex mappingVertex [[stage_in
4747
textureUV.sample(colorSampler, mappingVertex.textureCoordinate).rg, 1.0);
4848
return ycbcrToRGBTransform * ycbcr;
4949
}
50+
51+
kernel void writeRGBFromYUV(texture2d<uint, access::read> yTexture [[texture(0)]],
52+
texture2d<uint, access::read> uTexture [[texture(1)]],
53+
texture2d<uint, access::read> vTexture [[texture(2)]],
54+
texture2d<float, access::write> rgbTexture [[texture(3)]],
55+
uint2 yPosition [[thread_position_in_grid]])
56+
{
57+
float3x3 yuvToRGBTransform = float3x3(float3(+1.0000f, +1.0000f, +1.0000f),
58+
float3(+0.0000f, -0.3441f, +1.7720f),
59+
float3(+1.4020f, -0.7141f, +0.0000f));
60+
61+
uint2 uvPosition = uint2(yPosition.x / 2, yPosition.y / 2);
62+
63+
float3 yuvMatrix = float3(yTexture.read(yPosition).r / 255.0,
64+
uTexture.read(uvPosition).r / 255.0 - 0.5,
65+
vTexture.read(uvPosition).r / 255.0 - 0.5);
66+
67+
float3 rgbMatrix = yuvToRGBTransform * yuvMatrix;
68+
69+
rgbTexture.write(float4(float3(rgbMatrix), 1.0), yPosition);
70+
}

0 commit comments

Comments
 (0)