Skip to content

Commit 3eea077

Browse files
author
zhaoyongqiang
committed
add custom video encode capture
1 parent 7d4dd8e commit 3eea077

11 files changed

Lines changed: 176 additions & 36 deletions

File tree

‎iOS/APIExample/APIExample/Common/Utils/MediaUtils.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ NS_ASSUME_NONNULL_BEGIN
1414
@interface MediaUtils : NSObject
1515

1616
+ (CVPixelBufferRef)i420ToPixelBuffer:(void *)srcY srcU:(void *)srcU srcV:(void *)srcV width:(int)width height:(int)height;
17+
18+
+ (NSData *)dataFromPixelBuffer:(CVPixelBufferRef)pixelBuffer;
19+
1720
+ (nullable UIImage *)i420ToImage:(nullable void *)srcY srcU:(nullable void *)srcU srcV:(nullable void *)srcV width:(int)width height:(int)height;
1821
+ (nullable UIImage *)pixelBufferToImage:(CVPixelBufferRef)pixelBuffer;
1922

‎iOS/APIExample/APIExample/Common/Utils/MediaUtils.m‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,26 @@ + (CVPixelBufferRef)i420ToPixelBuffer:(void *)srcY srcU:(void *)srcU srcV:(void
6262
}
6363

6464

65+
+ (NSData *)dataFromPixelBuffer:(CVPixelBufferRef)pixelBuffer {
66+
NSMutableData* YUVMutData;
67+
CVPixelBufferLockBaseAddress(pixelBuffer, 0);
68+
unsigned char* yBaseAddress = (unsigned char*)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);
69+
int32_t yBytesPerRow = (int32_t)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
70+
int32_t width __unused = (int32_t)CVPixelBufferGetWidth(pixelBuffer);
71+
int32_t height = (int32_t)CVPixelBufferGetHeight(pixelBuffer);
72+
int32_t yLength = yBytesPerRow*height;
73+
YUVMutData = [NSMutableData dataWithBytes: yBaseAddress length: yLength];
74+
75+
unsigned char* uvBaseAddress = (unsigned char*)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1);
76+
int32_t uvBytesPerRow = (int32_t)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
77+
int32_t uvLength = uvBytesPerRow*height/2;
78+
NSMutableData* UVData = [NSMutableData dataWithBytes: uvBaseAddress length: uvLength];
79+
[YUVMutData appendData:UVData];
80+
NSData* YUVData = [NSData dataWithData:YUVMutData];
81+
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
82+
return YUVData;
83+
}
84+
6585
+ (UIImage *)i420ToImage:(void *)srcY srcU:(void *)srcU srcV:(void *)srcV width:(int)width height:(int)height {
6686
int size = width * height * 3 / 2;
6787
int yLength = width * height;

‎iOS/APIExample/APIExample/Examples/Advanced/CustomVideoSourcePushMulti/CustomVideoSourcePushMulti.swift‎

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ class UserModel {
1616
var trackId: UInt32 = 0
1717
var isJoin: Bool = false
1818
var customSource: AgoraYUVImageSourcePush?
19+
var isEncode: Bool = false
1920
}
2021

2122
class CustomVideoSourcePushMultiEntry : UIViewController
@@ -111,16 +112,19 @@ class CustomVideoSourcePushMultiMain: BaseViewController {
111112
// 2. If app certificate is turned on at dashboard, token is needed
112113
// when joining channel. The channel name and uid used to calculate
113114
// the token has to match the ones used for channel join
114-
joinChannel(uid: 999, trackId: customCamera?.trackId ?? 0)
115+
joinChannel(uid: 999,
116+
trackId: customCamera?.trackId ?? 0,
117+
publishEncodedVideoTrack: false)
115118
}
116-
private func joinChannel(uid: UInt, trackId: UInt32) {
119+
private func joinChannel(uid: UInt, trackId: UInt32, publishEncodedVideoTrack: Bool) {
117120
guard let channelName = configs["channelName"] as? String else {return}
118121
let option = AgoraRtcChannelMediaOptions()
119-
option.publishCustomVideoTrack = true
122+
option.publishCustomVideoTrack = !publishEncodedVideoTrack
120123
option.publishMicrophoneTrack = false
121124
option.autoSubscribeAudio = true
122125
option.autoSubscribeVideo = true
123126
option.customVideoTrackId = Int(trackId)
127+
option.publishEncodedVideoTrack = publishEncodedVideoTrack
124128
option.clientRoleType = GlobalSettings.shared.getUserRole()
125129
let connection = AgoraRtcConnection()
126130
connection.localUid = uid
@@ -146,13 +150,15 @@ class CustomVideoSourcePushMultiMain: BaseViewController {
146150
if userModel.trackId == 0 {
147151
let encodeVideoTrack = AgoraEncodedVideoTrackOptions()
148152
userModel.trackId = agoraKit.createCustomEncodedVideoTrack(encodeVideoTrack)
153+
userModel.isEncode = true
149154
}
150155
createVideoTrack(userModel: userModel)
151156
}
152157
@IBAction func onCreateVideoTrack(_ sender: Any) {
153158
guard let userModel = remoteVideos.first(where: { $0.isJoin == false }) else { return }
154159
if userModel.trackId == 0 {
155160
userModel.trackId = agoraKit.createCustomVideoTrack()
161+
userModel.isEncode = false
156162
}
157163
createVideoTrack(userModel: userModel)
158164
}
@@ -166,7 +172,9 @@ class CustomVideoSourcePushMultiMain: BaseViewController {
166172
userModel.isJoin = true
167173
userModel.customSource = customCamera
168174
customCamera.startSource()
169-
joinChannel(uid: userModel.uid, trackId: userModel.trackId)
175+
joinChannel(uid: userModel.uid,
176+
trackId: userModel.trackId,
177+
publishEncodedVideoTrack: userModel.isEncode)
170178
}
171179

172180
@IBAction func onDestoryVideoTrack(_ sender: Any) {
@@ -179,6 +187,9 @@ class CustomVideoSourcePushMultiMain: BaseViewController {
179187
let connection = AgoraRtcConnection()
180188
connection.localUid = userModel?.uid ?? 0
181189
connection.channelId = channelName
190+
userModel?.trackId = 0
191+
userModel?.isEncode = false
192+
userModel?.uid = UInt(Int.random(in: 10000...99999))
182193
agoraKit.leaveChannelEx(connection) { state in
183194
LogUtils.log(message: "warning: \(state.description)", level: .info)
184195
}
@@ -299,8 +310,6 @@ extension CustomVideoSourcePushMultiMain: AgoraYUVImageSourcePushDelegate {
299310
videoFrame.format = 12
300311
videoFrame.textureBuf = buffer
301312
videoFrame.rotation = Int32(rotation)
302-
//once we have the video frame, we can push to agora sdk
303-
agoraKit?.pushExternalVideoFrame(videoFrame, videoTrackId: trackId)
304313

305314
let outputVideoFrame = AgoraOutputVideoFrame()
306315
outputVideoFrame.width = Int32(size.width)
@@ -312,6 +321,20 @@ extension CustomVideoSourcePushMultiMain: AgoraYUVImageSourcePushDelegate {
312321
} else {
313322
let userModel = remoteVideos.first(where: { $0.trackId == trackId })
314323
userModel?.canvasView?.videoView.renderVideoPixelBuffer(outputVideoFrame)
324+
325+
if userModel?.isEncode == true {
326+
let data = MediaUtils.data(from: buffer)
327+
let info = AgoraEncodedVideoFrameInfo()
328+
info.frameType = .keyFrame
329+
info.framesPerSecond = 30
330+
info.codecType = .H264
331+
agoraKit.pushExternalEncodedVideoFrameEx(data,
332+
info: info,
333+
videoTrackId: trackId)
334+
return
335+
}
315336
}
337+
//once we have the video frame, we can push to agora sdk
338+
agoraKit?.pushExternalVideoFrame(videoFrame, videoTrackId: trackId)
316339
}
317340
}

‎macOS/APIExample/Common/NetworkManager/NetworkManager.swift‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,26 @@ class NetworkManager {
6767
}
6868
}
6969

70+
func download(urlString: String, success: SuccessClosure?, failure: FailClosure?) {
71+
let session = URLSession(configuration: sessionConfig)
72+
let request = URLRequest(url: URL(string: urlString) ?? URL(fileURLWithPath: urlString))
73+
let fileName = urlString.components(separatedBy: "/").last ?? ""
74+
let documnets = NSTemporaryDirectory() + fileName
75+
if FileManager.default.fileExists(atPath: documnets) {
76+
success?(["fileName": fileName, "path": documnets])
77+
return
78+
}
79+
DispatchQueue.global().async {
80+
let downloadTask = session.downloadTask(with: request) { location, response, error in
81+
let locationPath = location!.path
82+
let fileManager = FileManager.default
83+
try? fileManager.moveItem(atPath: locationPath, toPath: documnets)
84+
success?(["fileName": fileName, "path": documnets])
85+
}
86+
downloadTask.resume()
87+
}
88+
}
89+
7090
private func request(urlString: String,
7191
params: [String: Any]?,
7292
method: HTTPMethods,

‎macOS/APIExample/Common/Utils/MediaUtils.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ NS_ASSUME_NONNULL_BEGIN
1818

1919
+ (NSImage *)pixelBufferToImage: (CVPixelBufferRef)pixelBuffer;
2020

21+
+ (NSData *)dataFromPixelBuffer:(CVPixelBufferRef)pixelBuffer;
22+
2123
+ (nullable NSImage *)i420ToImage:(nullable void *)srcY srcU:(nullable void *)srcU srcV:(nullable void *)srcV width:(int)width height:(int)height;
2224

2325
@end

‎macOS/APIExample/Common/Utils/MediaUtils.m‎

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ + (CVPixelBufferRef)i420ToPixelBuffer:(void *)srcY srcU:(void *)srcU srcV:(void
4343
free(NV12buf);
4444
return nil;
4545
}
46-
46+
4747
CVPixelBufferLockBaseAddress(pixelBuffer,0);
4848
void *yDestPlane = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);
4949

@@ -52,7 +52,7 @@ + (CVPixelBufferRef)i420ToPixelBuffer:(void *)srcY srcU:(void *)srcU srcV:(void
5252
unsigned char *y_ch1 = NV12buf + w * h;
5353
memcpy(yDestPlane, y_ch0, w * h);
5454
void *uvDestPlane = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1);
55-
55+
5656
// Here y_ch1 is UV-Plane of YUV(NV12) data.
5757
memcpy(uvDestPlane, y_ch1, w * h * 0.5);
5858
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
@@ -61,6 +61,27 @@ + (CVPixelBufferRef)i420ToPixelBuffer:(void *)srcY srcU:(void *)srcU srcV:(void
6161
return pixelBuffer;
6262
}
6363

64+
65+
+ (NSData *)dataFromPixelBuffer:(CVPixelBufferRef)pixelBuffer {
66+
NSMutableData* YUVMutData;
67+
CVPixelBufferLockBaseAddress(pixelBuffer, 0);
68+
unsigned char* yBaseAddress = (unsigned char*)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);
69+
int32_t yBytesPerRow = (int32_t)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
70+
int32_t width __unused = (int32_t)CVPixelBufferGetWidth(pixelBuffer);
71+
int32_t height = (int32_t)CVPixelBufferGetHeight(pixelBuffer);
72+
int32_t yLength = yBytesPerRow*height;
73+
YUVMutData = [NSMutableData dataWithBytes: yBaseAddress length: yLength];
74+
75+
unsigned char* uvBaseAddress = (unsigned char*)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1);
76+
int32_t uvBytesPerRow = (int32_t)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
77+
int32_t uvLength = uvBytesPerRow*height/2;
78+
NSMutableData* UVData = [NSMutableData dataWithBytes: uvBaseAddress length: uvLength];
79+
[YUVMutData appendData:UVData];
80+
NSData* YUVData = [NSData dataWithData:YUVMutData];
81+
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
82+
return YUVData;
83+
}
84+
6485
+ (NSImage *)i420ToImage:(void *)srcY srcU:(void *)srcU srcV:(void *)srcV width:(int)width height:(int)height {
6586
int size = width * height * 3 / 2;
6687
int yLength = width * height;
@@ -91,7 +112,7 @@ + (NSImage *)i420ToImage:(void *)srcY srcU:(void *)srcU srcV:(void *)srcV width:
91112
free(NV12buf);
92113
return nil;
93114
}
94-
115+
95116
CVPixelBufferLockBaseAddress(pixelBuffer,0);
96117
void *yDestPlane = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);
97118

@@ -100,7 +121,7 @@ + (NSImage *)i420ToImage:(void *)srcY srcU:(void *)srcU srcV:(void *)srcV width:
100121
unsigned char *y_ch1 = NV12buf + w * h;
101122
memcpy(yDestPlane, y_ch0, w * h);
102123
void *uvDestPlane = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1);
103-
124+
104125
// Here y_ch1 is UV-Plane of YUV(NV12) data.
105126
memcpy(uvDestPlane, y_ch1, w * h * 0.5);
106127
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
@@ -109,8 +130,8 @@ + (NSImage *)i420ToImage:(void *)srcY srcU:(void *)srcU srcV:(void *)srcV width:
109130
CIImage *coreImage = [CIImage imageWithCVPixelBuffer:pixelBuffer];
110131
CIContext *temporaryContext = [CIContext contextWithOptions:nil];
111132
CGImageRef videoImage = [temporaryContext createCGImage:coreImage
112-
fromRect:CGRectMake(0, 0, w, h)];
113-
133+
fromRect:CGRectMake(0, 0, w, h)];
134+
114135
NSImage *finalImage = [[NSImage alloc] initWithCGImage:videoImage size:NSMakeSize(width, height)];
115136
CVPixelBufferRelease(pixelBuffer);
116137
CGImageRelease(videoImage);
@@ -129,7 +150,7 @@ + (NSImage *)pixelBufferToImage: (CVPixelBufferRef)pixelBuffer {
129150
fromRect:CGRectMake(0, 0, height, width)];
130151

131152
NSImage *finalImage = [[NSImage alloc] initWithCGImage:videoImage size: CGSizeMake(width, height)];
132-
153+
133154
// CVPixelBufferRelease(pixelBuffer);
134155
CGImageRelease(videoImage);
135156
return finalImage;

‎macOS/APIExample/Examples/Advanced/CustomVideoSourcePushMulti/Base.lproj/CustomVideoSourcePushMulti.storyboard‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
<?xml version="1.0" encoding="UTF-8"?>
2-
<document type="com.apple.InterfaceBuilder3.Cocoa.Storyboard.XIB" version="3.0" toolsVersion="21507" targetRuntime="MacOSX.Cocoa" propertyAccessControl="none" useAutolayout="YES">
2+
<document type="com.apple.InterfaceBuilder3.Cocoa.Storyboard.XIB" version="3.0" toolsVersion="21701" targetRuntime="MacOSX.Cocoa" propertyAccessControl="none" useAutolayout="YES">
33
<dependencies>
44
<deployment identifier="macosx"/>
5-
<plugIn identifier="com.apple.InterfaceBuilder.CocoaPlugin" version="21507"/>
5+
<plugIn identifier="com.apple.InterfaceBuilder.CocoaPlugin" version="21701"/>
66
<capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
77
</dependencies>
88
<scenes>
@@ -37,7 +37,7 @@
3737
<autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMinY="YES"/>
3838
<buttonCell key="cell" type="push" title="Create VideoTrack" bezelStyle="rounded" alignment="center" borderStyle="border" imageScaling="proportionallyDown" inset="2" id="tbi-87-lo2">
3939
<behavior key="behavior" pushIn="YES" lightByBackground="YES" lightByGray="YES"/>
40-
<font key="font" size="13" name=".PingFangSC-Regular"/>
40+
<font key="font" metaFont="system"/>
4141
</buttonCell>
4242
<connections>
4343
<action selector="onClickCreateTrack:" target="sXF-vm-Rrb" id="Tfj-aG-hqw"/>
@@ -46,14 +46,25 @@
4646
<button verticalHuggingPriority="750" fixedFrame="YES" translatesAutoresizingMaskIntoConstraints="NO" id="cJB-Ul-7E2">
4747
<rect key="frame" x="160" y="232" width="158" height="32"/>
4848
<autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMinY="YES"/>
49-
<buttonCell key="cell" type="push" title="Destory VideoTrack" bezelStyle="rounded" alignment="center" borderStyle="border" imageScaling="proportionallyDown" inset="2" id="fY0-1R-m7R">
49+
<buttonCell key="cell" type="push" title="Destory Track" bezelStyle="rounded" alignment="center" borderStyle="border" imageScaling="proportionallyDown" inset="2" id="fY0-1R-m7R">
5050
<behavior key="behavior" pushIn="YES" lightByBackground="YES" lightByGray="YES"/>
51-
<font key="font" size="13" name=".PingFangSC-Regular"/>
51+
<font key="font" metaFont="system"/>
5252
</buttonCell>
5353
<connections>
5454
<action selector="onClickDestoryTrack:" target="sXF-vm-Rrb" id="iPM-9C-fRn"/>
5555
</connections>
5656
</button>
57+
<button verticalHuggingPriority="750" fixedFrame="YES" translatesAutoresizingMaskIntoConstraints="NO" id="50e-J7-EcM">
58+
<rect key="frame" x="-1" y="273" width="206" height="32"/>
59+
<autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMinY="YES"/>
60+
<buttonCell key="cell" type="push" title="Create Video Track(Encode)" bezelStyle="rounded" alignment="center" borderStyle="border" imageScaling="proportionallyDown" inset="2" id="c9C-Yd-1NT">
61+
<behavior key="behavior" pushIn="YES" lightByBackground="YES" lightByGray="YES"/>
62+
<font key="font" metaFont="system"/>
63+
</buttonCell>
64+
<connections>
65+
<action selector="onClickCustomEncodeVideoTrack:" target="sXF-vm-Rrb" id="TJu-OS-vpm"/>
66+
</connections>
67+
</button>
5768
</subviews>
5869
</view>
5970
</subviews>

0 commit comments

Comments
 (0)