Skip to content

Commit a9ab6ff

Browse files
youennfyouennf
committed
Add support for RTCEncodedFrame metadata
https://bugs.webkit.org/show_bug.cgi?id=224430 Reviewed by Eric Carlson. Source/WebCore: Expose metadata given from backend. Audio sender side uses a regular TransformableFrameInterface instead of a TransformableAudioFrameInterface. Make sure to not call GetHeader() in that case. Test: http/wpt/webrtc/metadata-transform.html * Modules/mediastream/RTCEncodedAudioFrame.cpp: (WebCore::RTCEncodedAudioFrame::timestamp const): (WebCore::RTCEncodedAudioFrame::getMetadata): * Modules/mediastream/RTCEncodedAudioFrame.h: * Modules/mediastream/RTCEncodedAudioFrame.idl: * Modules/mediastream/RTCEncodedVideoFrame.cpp: (WebCore::RTCEncodedVideoFrame::timestamp const): (WebCore::RTCEncodedVideoFrame::getMetadata): * Modules/mediastream/RTCEncodedVideoFrame.h: * Modules/mediastream/RTCEncodedVideoFrame.idl: * Modules/mediastream/RTCRtpTransformableFrame.h: * Modules/mediastream/libwebrtc/LibWebRTCRtpTransformBackend.cpp: (WebCore::LibWebRTCRtpTransformBackend::Transform): * Modules/mediastream/libwebrtc/LibWebRTCRtpTransformableFrame.cpp: (WebCore::LibWebRTCRtpTransformableFrame::LibWebRTCRtpTransformableFrame): (WebCore::LibWebRTCRtpTransformableFrame::timestamp const): (WebCore::LibWebRTCRtpTransformableFrame::audioMetadata const): (WebCore::LibWebRTCRtpTransformableFrame::videoMetadata const): * Modules/mediastream/libwebrtc/LibWebRTCRtpTransformableFrame.h: LayoutTests: * http/wpt/webrtc/metadata-transform-expected.txt: Added. * http/wpt/webrtc/metadata-transform-worker.js: Added. (onrtctransform.process): (onrtctransform): * http/wpt/webrtc/metadata-transform.html: Added. Canonical link: https://commits.webkit.org/236400@main git-svn-id: https://svn.webkit.org/repository/webkit/trunk@275830 268f45cc-cd09-0410-ab3c-d52691b4dbfc
1 parent ef99370 commit a9ab6ff

15 files changed

Lines changed: 297 additions & 8 deletions

‎LayoutTests/ChangeLog‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,16 @@
1+
2021-04-12 Youenn Fablet <youenn@apple.com>
2+
3+
Add support for RTCEncodedFrame metadata
4+
https://bugs.webkit.org/show_bug.cgi?id=224430
5+
6+
Reviewed by Eric Carlson.
7+
8+
* http/wpt/webrtc/metadata-transform-expected.txt: Added.
9+
* http/wpt/webrtc/metadata-transform-worker.js: Added.
10+
(onrtctransform.process):
11+
(onrtctransform):
12+
* http/wpt/webrtc/metadata-transform.html: Added.
13+
114
2021-04-12 Philippe Normand <pnormand@igalia.com>
215

316
[GStreamer][WebRTC] An audio track should be muted when capture is interrupted by the OS.

‎LayoutTests/http/wpt/webrtc/metadata-transform-expected.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
2+
3+
PASS audio exchange with transform
4+
PASS video exchange with transform
5+

‎LayoutTests/http/wpt/webrtc/metadata-transform-worker.js‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
onrtctransform = (event) => {
2+
const transformer = event.transformer;
3+
transformer.port.onmessage = (event) => transformer.port.postMessage(event.data);
4+
5+
transformer.reader = transformer.readable.getReader();
6+
transformer.writer = transformer.writable.getWriter();
7+
8+
let isFirstFrame = true;
9+
function process(transformer)
10+
{
11+
transformer.reader.read().then(chunk => {
12+
if (chunk.done)
13+
return;
14+
15+
if (isFirstFrame) {
16+
isFirstFrame = false;
17+
self.postMessage({ name: transformer.options.name, timestamp: chunk.value.timestamp, metadata: chunk.value.getMetadata() });
18+
}
19+
transformer.writer.write(chunk.value);
20+
process(transformer);
21+
});
22+
}
23+
process(transformer);
24+
};
25+
self.postMessage("registered");

‎LayoutTests/http/wpt/webrtc/metadata-transform.html‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
<!doctype html>
2+
<html>
3+
<head>
4+
<meta charset="utf-8">
5+
<script src="/resources/testharness.js"></script>
6+
<script src="/resources/testharnessreport.js"></script>
7+
</head>
8+
<body>
9+
<video id="video1" autoplay></video>
10+
<script src ="routines.js"></script>
11+
<script>
12+
async function waitForMessage(worker, data)
13+
{
14+
while (true) {
15+
const received = await new Promise(resolve => worker.onmessage = (event) => resolve(event.data));
16+
if (data === received)
17+
return;
18+
}
19+
}
20+
21+
async function gatherMetadata(audio)
22+
{
23+
worker = new Worker('metadata-transform-worker.js');
24+
const data = await new Promise(resolve => worker.onmessage = (event) => resolve(event.data));
25+
assert_equals(data, "registered");
26+
27+
const localStream = await navigator.mediaDevices.getUserMedia({audio: audio, video: !audio});
28+
29+
let sender, receiver;
30+
const senderTransform = new RTCRtpScriptTransform(worker, {name:'sender'});
31+
const receiverTransform = new RTCRtpScriptTransform(worker, {name:'receiver'});
32+
33+
await new Promise((resolve, reject) => {
34+
createConnections((firstConnection) => {
35+
pc1 = firstConnection;
36+
sender = firstConnection.addTrack(localStream.getTracks()[0], localStream);
37+
sender.transform = senderTransform;
38+
}, (secondConnection) => {
39+
pc2 = secondConnection;
40+
secondConnection.ontrack = (trackEvent) => {
41+
receiver = trackEvent.receiver;
42+
receiver.transform = receiverTransform;
43+
resolve(trackEvent.streams[0]);
44+
};
45+
});
46+
setTimeout(() => reject("Test timed out"), 5000);
47+
});
48+
49+
return new Promise((resolve, reject) => {
50+
let senderMetadata, senderTimestamp;
51+
worker.onmessage = (event) => {
52+
if (event.data.name === 'sender') {
53+
senderMetadata = event.data.metadata;
54+
senderTimestamp = event.data.timestamp;
55+
} else if (event.data.name === 'receiver')
56+
resolve([senderMetadata, senderTimestamp, event.data.metadata, event.data.timestamp]);
57+
};
58+
setTimeout(() => reject("Metadata test timed out"), 5000);
59+
});
60+
}
61+
62+
promise_test(async (test) => {
63+
const [senderMetadata, senderTimestamp, receiverMetadata, receiverTimestamp] = await gatherMetadata(true);
64+
65+
assert_equals(senderTimestamp, receiverTimestamp, "timestamp");
66+
assert_true(!!senderMetadata.synchronizationSource, "ssrc");
67+
assert_equals(senderMetadata.synchronizationSource, receiverMetadata.synchronizationSource, "ssrc");
68+
assert_array_equals(senderMetadata.contributingSources, receiverMetadata.contributingSources, "csrc");
69+
}, "audio exchange with transform");
70+
71+
promise_test(async (test) => {
72+
const [senderMetadata, senderTimestamp, receiverMetadata, receiverTimestamp] = await gatherMetadata(true);
73+
74+
assert_equals(senderTimestamp, receiverTimestamp, "timestamp");
75+
assert_true(!!senderMetadata.synchronizationSource, "ssrc");
76+
assert_equals(senderMetadata.synchronizationSource, receiverMetadata.synchronizationSource, "ssrc");
77+
assert_array_equals(senderMetadata.contributingSources, receiverMetadata.contributingSources, "csrc");
78+
assert_equals(senderMetadata.height, receiverMetadata.height, "height");
79+
assert_equals(senderMetadata.width, receiverMetadata.width, "width");
80+
assert_equals(senderMetadata.spatialIndex, receiverMetadata.spatialIndex, "spatialIndex");
81+
assert_equals(senderMetadata.temporalIndex, receiverMetadata.temporalIndex, "temporalIndex");
82+
}, "video exchange with transform");
83+
</script>
84+
</body>
85+
</html>

‎Source/WebCore/ChangeLog‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,36 @@
1+
2021-04-12 Youenn Fablet <youenn@apple.com>
2+
3+
Add support for RTCEncodedFrame metadata
4+
https://bugs.webkit.org/show_bug.cgi?id=224430
5+
6+
Reviewed by Eric Carlson.
7+
8+
Expose metadata given from backend.
9+
Audio sender side uses a regular TransformableFrameInterface instead of a TransformableAudioFrameInterface.
10+
Make sure to not call GetHeader() in that case.
11+
12+
Test: http/wpt/webrtc/metadata-transform.html
13+
14+
* Modules/mediastream/RTCEncodedAudioFrame.cpp:
15+
(WebCore::RTCEncodedAudioFrame::timestamp const):
16+
(WebCore::RTCEncodedAudioFrame::getMetadata):
17+
* Modules/mediastream/RTCEncodedAudioFrame.h:
18+
* Modules/mediastream/RTCEncodedAudioFrame.idl:
19+
* Modules/mediastream/RTCEncodedVideoFrame.cpp:
20+
(WebCore::RTCEncodedVideoFrame::timestamp const):
21+
(WebCore::RTCEncodedVideoFrame::getMetadata):
22+
* Modules/mediastream/RTCEncodedVideoFrame.h:
23+
* Modules/mediastream/RTCEncodedVideoFrame.idl:
24+
* Modules/mediastream/RTCRtpTransformableFrame.h:
25+
* Modules/mediastream/libwebrtc/LibWebRTCRtpTransformBackend.cpp:
26+
(WebCore::LibWebRTCRtpTransformBackend::Transform):
27+
* Modules/mediastream/libwebrtc/LibWebRTCRtpTransformableFrame.cpp:
28+
(WebCore::LibWebRTCRtpTransformableFrame::LibWebRTCRtpTransformableFrame):
29+
(WebCore::LibWebRTCRtpTransformableFrame::timestamp const):
30+
(WebCore::LibWebRTCRtpTransformableFrame::audioMetadata const):
31+
(WebCore::LibWebRTCRtpTransformableFrame::videoMetadata const):
32+
* Modules/mediastream/libwebrtc/LibWebRTCRtpTransformableFrame.h:
33+
134
2021-04-12 Sam Weinig <weinig@apple.com>
235

336
Remove unused 'DefaultDefineOwnProperty' extended attribute support

‎Source/WebCore/Modules/mediastream/RTCEncodedAudioFrame.cpp‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,18 @@ RTCEncodedAudioFrame::RTCEncodedAudioFrame(Ref<RTCRtpTransformableFrame>&& frame
3737

3838
RTCEncodedAudioFrame::~RTCEncodedAudioFrame() = default;
3939

40+
uint64_t RTCEncodedAudioFrame::timestamp() const
41+
{
42+
return m_frame->timestamp();
43+
}
44+
45+
const RTCEncodedAudioFrame::Metadata& RTCEncodedAudioFrame::getMetadata()
46+
{
47+
if (!m_metadata)
48+
m_metadata = m_frame->audioMetadata();
49+
return *m_metadata;
50+
}
51+
4052
} // namespace WebCore
4153

4254
#endif // ENABLE(WEB_RTC)

‎Source/WebCore/Modules/mediastream/RTCEncodedAudioFrame.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,15 @@ class RTCEncodedAudioFrame : public RTCEncodedFrame {
3636
static Ref<RTCEncodedAudioFrame> create(Ref<RTCRtpTransformableFrame>&& frame) { return adoptRef(*new RTCEncodedAudioFrame(WTFMove(frame))); }
3737
~RTCEncodedAudioFrame();
3838

39+
using Metadata = RTCEncodedAudioFrameMetadata;
40+
const Metadata& getMetadata();
41+
42+
uint64_t timestamp() const;
43+
3944
private:
4045
explicit RTCEncodedAudioFrame(Ref<RTCRtpTransformableFrame>&&);
46+
47+
mutable Optional<Metadata> m_metadata;
4148
};
4249

4350
} // namespace WebCore

‎Source/WebCore/Modules/mediastream/RTCEncodedAudioFrame.idl‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,22 @@
2323
* THE POSSIBILITY OF SUCH DAMAGE.
2424
*/
2525

26+
[
27+
Conditional=WEB_RTC,
28+
JSGenerateToJSObject,
29+
]
30+
dictionary RTCEncodedAudioFrameMetadata {
31+
unsigned long synchronizationSource;
32+
sequence<unsigned long> contributingSources;
33+
};
34+
2635
[
2736
Conditional=WEB_RTC,
2837
EnabledBySetting=WebRTCInsertableStreams,
2938
Exposed=DedicatedWorker,
3039
ImplementationLacksVTable,
3140
] interface RTCEncodedAudioFrame {
32-
// readonly attribute unsigned long long timestamp;
41+
readonly attribute unsigned long long timestamp;
3342
attribute ArrayBuffer data;
34-
// RTCAudioFrameMetadata getMetadata();
43+
RTCEncodedAudioFrameMetadata getMetadata();
3544
};

‎Source/WebCore/Modules/mediastream/RTCEncodedVideoFrame.cpp‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,18 @@ RTCEncodedVideoFrame::RTCEncodedVideoFrame(Ref<RTCRtpTransformableFrame>&& frame
3838

3939
RTCEncodedVideoFrame::~RTCEncodedVideoFrame() = default;
4040

41+
uint64_t RTCEncodedVideoFrame::timestamp() const
42+
{
43+
return m_frame->timestamp();
44+
}
45+
46+
const RTCEncodedVideoFrame::Metadata& RTCEncodedVideoFrame::getMetadata()
47+
{
48+
if (!m_metadata)
49+
m_metadata = m_frame->videoMetadata();
50+
return *m_metadata;
51+
}
52+
4153
} // namespace WebCore
4254

4355
#endif // ENABLE(WEB_RTC)

‎Source/WebCore/Modules/mediastream/RTCEncodedVideoFrame.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,16 @@ class RTCEncodedVideoFrame : public RTCEncodedFrame {
3939
enum class Type { Empty, Key, Delta };
4040
Type type() const { return m_type; }
4141

42+
using Metadata = RTCEncodedVideoFrameMetadata;
43+
const Metadata& getMetadata();
44+
45+
uint64_t timestamp() const;
46+
4247
private:
4348
explicit RTCEncodedVideoFrame(Ref<RTCRtpTransformableFrame>&&);
4449

4550
Type m_type;
51+
mutable Optional<Metadata> m_metadata;
4652
};
4753

4854
} // namespace WebCore

0 commit comments

Comments
 (0)