Skip to content

Commit 5f6aba2

Browse files
vvfedorenkofacebook-github-bot
vvfedorenko
authored and
facebook-github-bot
committed
Implementation of FBCLock v2
Summary: Implementation of FBClock v2 based on approximation of PHC time with MONOTONIC_RAW system clock and fallback to REALTIME if not supported by the kernel Reviewed By: t3lurid3, abulimov Differential Revision: D75152073 fbshipit-source-id: 8839389d7c52fa7cc9b8eda01ab2fb2d3a77b208
1 parent 2aede5e commit 5f6aba2

5 files changed

Lines changed: 441 additions & 12 deletions

File tree

‎fbclock/cpp_test/test.cpp‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,99 @@ TEST(fbclockTest, test_concurrent) {
145145
remove(test_shm);
146146
}
147147

148+
int writer_thread_v2(int sfd_rw, int tries) {
149+
int err;
150+
fbclock_clockdata_v2 data = {
151+
.ingress_time_ns = 1,
152+
.error_bound_ns = 2,
153+
.holdover_multiplier_ns = 3,
154+
.phc_time_ns = 1748164346441310791,
155+
.sysclock_time_ns = 1748164309441310791,
156+
.clockId = CLOCK_MONOTONIC_RAW,
157+
};
158+
for (int i = 0; i < tries; i++) {
159+
err = fbclock_clockdata_store_data_v2(sfd_rw, &data);
160+
if (err != 0) {
161+
return err;
162+
}
163+
data.ingress_time_ns = data.ingress_time_ns + 1000;
164+
if (data.ingress_time_ns > 10000) {
165+
data.ingress_time_ns = 1;
166+
}
167+
data.error_bound_ns = data.ingress_time_ns * 2;
168+
data.holdover_multiplier_ns = data.ingress_time_ns * 3;
169+
data.phc_time_ns += 10000;
170+
data.sysclock_time_ns += 10000;
171+
usleep(10000); // sleep 10ms - this will be the normal case
172+
}
173+
return 0;
174+
}
175+
176+
int reader_thread_v2(fbclock_shmdata_v2* shmp, int tries) {
177+
int err;
178+
fbclock_clockdata_v2 data;
179+
for (int i = 0; i < tries; i++) {
180+
err = fbclock_clockdata_load_data_v2(shmp, &data);
181+
if (err != 0) {
182+
return err;
183+
}
184+
if (data.ingress_time_ns * 2 != data.error_bound_ns) {
185+
printf("ingress_time_ns: %lu\n", data.ingress_time_ns);
186+
printf("error_bound_ns: %d\n", data.error_bound_ns);
187+
printf("holdover_multiplier_ns: %d\n", data.holdover_multiplier_ns);
188+
return -1;
189+
}
190+
if (data.ingress_time_ns * 3 != data.holdover_multiplier_ns) {
191+
printf("ingress_time_ns: %lu\n", data.ingress_time_ns);
192+
printf("error_bound_ns: %d\n", data.error_bound_ns);
193+
printf("holdover_multiplier_ns: %d\n", data.holdover_multiplier_ns);
194+
return -1;
195+
}
196+
if ((data.phc_time_ns - data.sysclock_time_ns) != 37000000000) {
197+
printf("phc_time_ns: %lu\n", data.phc_time_ns);
198+
printf("sysclock_time_ns: %lu\n", data.sysclock_time_ns);
199+
return -1;
200+
}
201+
}
202+
return 0;
203+
}
204+
205+
TEST(fbclockTest, test_concurrent_v2) {
206+
int err;
207+
char* test_shm = std::tmpnam(nullptr);
208+
209+
// open file, write data into it
210+
FILE* f_rw = fopen(test_shm, "wb+");
211+
int sfd_rw = fileno(f_rw);
212+
ASSERT_NE(sfd_rw, -1);
213+
214+
err = ftruncate(sfd_rw, FBCLOCK_SHMDATA_V2_SIZE);
215+
ASSERT_EQ(err, 0);
216+
217+
// read data from the file
218+
FILE* f_ro = fopen(test_shm, "r");
219+
int sfd_ro = fileno(f_ro);
220+
ASSERT_NE(sfd_ro, -1);
221+
222+
fbclock_shmdata_v2* shmp = (fbclock_shmdata_v2*)mmap(
223+
nullptr, FBCLOCK_SHMDATA_V2_SIZE, PROT_READ, MAP_SHARED, sfd_ro, 0);
224+
ASSERT_NE(shmp, MAP_FAILED);
225+
226+
int tries = 1000;
227+
228+
// spawn two functions asynchronously, make sure there is no inconsistent data
229+
auto future_writer =
230+
std::async(std::launch::async, writer_thread_v2, sfd_rw, tries);
231+
auto future_reader =
232+
std::async(std::launch::async, reader_thread_v2, shmp, tries * 10);
233+
err = future_writer.get();
234+
ASSERT_EQ(err, 0);
235+
err = future_reader.get();
236+
ASSERT_EQ(err, 0);
237+
munmap(shmp, FBCLOCK_SHMDATA_V2_SIZE);
238+
remove(test_shm);
239+
}
240+
148241
TEST(fbclockTest, test_window_of_uncertainty) {
149242
int64_t seconds = 0; // how long ago was the last SYNC
150243
double error_bound_ns = 172.0;

‎fbclock/fbclock.c‎

Lines changed: 191 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ limitations under the License.
2323
#include <sys/ioctl.h>
2424
#include <sys/mman.h>
2525
#include <sys/stat.h>
26-
#include <sys/types.h>
26+
#include <time.h>
2727
#include <unistd.h> // close
2828
#include "missing.h"
2929

@@ -37,6 +37,7 @@ limitations under the License.
3737
#endif
3838

3939
#define FBCLOCK_CLOCKDATA_SIZE sizeof(fbclock_clockdata)
40+
#define FBCLOCK_CLOCKDATA_V2_SIZE sizeof(fbclock_clockdata_v2)
4041
#define FBCLOCK_MAX_READ_TRIES 1000
4142
#define NANOSECONDS_IN_SECONDS 1e9
4243

@@ -67,6 +68,16 @@ static inline uint64_t fbclock_clockdata_crc(fbclock_clockdata* value) {
6768
return counter ^ 0xFFFFFFFF;
6869
}
6970

71+
int ends_with(const char* str, const char* suffix) {
72+
if (!str || !suffix)
73+
return 0;
74+
size_t lenstr = strlen(str);
75+
size_t lensuffix = strlen(suffix);
76+
if (lensuffix > lenstr)
77+
return 0;
78+
return strncmp(str + lenstr - lensuffix, suffix, lensuffix) == 0;
79+
}
80+
7081
int fbclock_clockdata_store_data(uint32_t fd, fbclock_clockdata* data) {
7182
fbclock_shmdata* shmp = mmap(
7283
NULL, FBCLOCK_SHMDATA_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
@@ -80,6 +91,24 @@ int fbclock_clockdata_store_data(uint32_t fd, fbclock_clockdata* data) {
8091
return FBCLOCK_E_NO_ERROR;
8192
}
8293

94+
int fbclock_clockdata_store_data_v2(uint32_t fd, fbclock_clockdata_v2* data) {
95+
fbclock_shmdata_v2* shmp = mmap(
96+
NULL, FBCLOCK_SHMDATA_V2_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
97+
if (shmp == MAP_FAILED) {
98+
return FBCLOCK_E_SHMEM_MAP_FAILED;
99+
}
100+
uint64_t seq = atomic_load(&shmp->seq);
101+
seq++;
102+
atomic_store(&shmp->seq, seq);
103+
__sync_synchronize();
104+
memcpy(&shmp->data, data, FBCLOCK_CLOCKDATA_V2_SIZE);
105+
__sync_synchronize();
106+
seq++;
107+
atomic_store(&shmp->seq, seq);
108+
munmap(shmp, FBCLOCK_SHMDATA_V2_SIZE);
109+
return FBCLOCK_E_NO_ERROR;
110+
}
111+
83112
int fbclock_clockdata_load_data(
84113
fbclock_shmdata* shmp,
85114
fbclock_clockdata* data) {
@@ -99,6 +128,28 @@ int fbclock_clockdata_load_data(
99128
return FBCLOCK_E_NO_ERROR;
100129
}
101130

131+
int fbclock_clockdata_load_data_v2(
132+
fbclock_shmdata_v2* shmp,
133+
fbclock_clockdata_v2* data) {
134+
for (int i = 0; i < FBCLOCK_MAX_READ_TRIES; i++) {
135+
uint64_t seq = atomic_load(&shmp->seq);
136+
if (seq & 1) {
137+
__sync_synchronize();
138+
continue;
139+
}
140+
__sync_synchronize();
141+
memcpy(data, &shmp->data, FBCLOCK_CLOCKDATA_V2_SIZE);
142+
__sync_synchronize();
143+
if (seq == atomic_load(&shmp->seq)) {
144+
fbclock_debug_print("reading clock data took %d tries\n", i + 1);
145+
return FBCLOCK_E_NO_ERROR;
146+
}
147+
}
148+
fbclock_debug_print(
149+
"failed to read clock data after %d tries\n", FBCLOCK_MAX_READ_TRIES);
150+
return FBCLOCK_E_CRC_MISMATCH;
151+
}
152+
102153
static inline int64_t fbclock_pct2ns(const struct ptp_clock_time* ptc) {
103154
return (int64_t)(ptc->sec * NANOSECONDS_IN_SECONDS) + (int64_t)ptc->nsec;
104155
}
@@ -177,17 +228,29 @@ int fbclock_init(fbclock_lib* lib, const char* shm_path) {
177228
lib->gettime = fbclock_read_ptp_offset;
178229
}
179230

180-
fbclock_shmdata* shmp =
181-
mmap(NULL, FBCLOCK_SHMDATA_SIZE, PROT_READ, MAP_SHARED, lib->shm_fd, 0);
182-
if (shmp == MAP_FAILED) {
183-
return FBCLOCK_E_SHMEM_MAP_FAILED;
231+
if (ends_with(shm_path, "_v2")) {
232+
fbclock_debug_print("Using v2 shared memory with path %s\n", shm_path);
233+
fbclock_shmdata_v2* shmp = mmap(
234+
NULL, FBCLOCK_SHMDATA_V2_SIZE, PROT_READ, MAP_SHARED, lib->shm_fd, 0);
235+
if (shmp == MAP_FAILED) {
236+
return FBCLOCK_E_SHMEM_MAP_FAILED;
237+
}
238+
lib->shmp_v2 = shmp;
239+
240+
} else {
241+
fbclock_shmdata* shmp =
242+
mmap(NULL, FBCLOCK_SHMDATA_SIZE, PROT_READ, MAP_SHARED, lib->shm_fd, 0);
243+
if (shmp == MAP_FAILED) {
244+
return FBCLOCK_E_SHMEM_MAP_FAILED;
245+
}
246+
lib->shmp = shmp;
184247
}
185-
lib->shmp = shmp;
186248
return FBCLOCK_E_NO_ERROR;
187249
}
188250

189251
int fbclock_destroy(fbclock_lib* lib) {
190252
munmap(lib->shmp, FBCLOCK_SHMDATA_SIZE);
253+
munmap(lib->shmp_v2, FBCLOCK_SHMDATA_V2_SIZE);
191254
close(lib->dev_fd);
192255
close(lib->shm_fd);
193256
return FBCLOCK_E_NO_ERROR;
@@ -235,10 +298,42 @@ int fbclock_calculate_time(
235298
return FBCLOCK_E_NO_ERROR;
236299
}
237300

301+
int fbclock_calculate_time_v2(
302+
uint64_t error_bound_ns,
303+
double h_value_ns,
304+
fbclock_clockdata_v2* state,
305+
int64_t phc_time_ns,
306+
int64_t sysclock_time_ns,
307+
int64_t sysclock_time_now_ns,
308+
fbclock_truetime* truetime,
309+
int time_standard) {
310+
if (state->ingress_time_ns > phc_time_ns) {
311+
return FBCLOCK_E_PHC_IN_THE_PAST;
312+
}
313+
// check how far back since last SYNC message from GM (in seconds)
314+
double seconds =
315+
(double)(phc_time_ns - state->ingress_time_ns) / NANOSECONDS_IN_SECONDS;
316+
317+
// UTC offset applied if time standard used is UTC (and not TAI)
318+
if (time_standard == FBCLOCK_UTC) {
319+
phc_time_ns = fbclock_apply_utc_offset_v2(state, phc_time_ns);
320+
}
321+
322+
int64_t diff_ns = sysclock_time_now_ns - sysclock_time_ns;
323+
phc_time_ns += diff_ns;
324+
325+
// calculate the Window of Uncertainty (WOU) (in nanoseconds)
326+
uint64_t wou_ns =
327+
fbclock_window_of_uncertainty(seconds, error_bound_ns, h_value_ns);
328+
truetime->earliest_ns = phc_time_ns - wou_ns;
329+
truetime->latest_ns = phc_time_ns + wou_ns;
330+
return FBCLOCK_E_NO_ERROR;
331+
}
332+
238333
int fbclock_gettime_tz(
239334
fbclock_lib* lib,
240335
fbclock_truetime* truetime,
241-
int timezone) {
336+
int time_standard) {
242337
struct phc_time_res res;
243338
fbclock_clockdata state = {};
244339
int rcode = fbclock_clockdata_load_data(lib->shmp, &state);
@@ -268,14 +363,66 @@ int fbclock_gettime_tz(
268363
double h_value = (double)state.holdover_multiplier_ns / FBCLOCK_POW2_16;
269364

270365
return fbclock_calculate_time(
271-
error_bound, h_value, &state, res.ts, truetime, timezone);
366+
error_bound, h_value, &state, res.ts, truetime, time_standard);
367+
}
368+
369+
int fbclock_gettime_tz_v2(
370+
fbclock_lib* lib,
371+
fbclock_truetime* truetime,
372+
int time_standard) {
373+
fbclock_clockdata_v2 state = {};
374+
int rcode = fbclock_clockdata_load_data_v2(lib->shmp_v2, &state);
375+
if (rcode != FBCLOCK_E_NO_ERROR) {
376+
return rcode;
377+
}
378+
379+
// cannot determine Truetime without these values
380+
if (state.error_bound_ns == 0 || state.ingress_time_ns == 0) {
381+
return FBCLOCK_E_NO_DATA;
382+
}
383+
if (state.phc_time_ns == 0 || state.sysclock_time_ns == 0) {
384+
return FBCLOCK_E_NO_DATA;
385+
}
386+
387+
// if the value is stored as UINT32_MAX then it's too big
388+
if (state.error_bound_ns == UINT32_MAX ||
389+
state.holdover_multiplier_ns == UINT32_MAX) {
390+
return FBCLOCK_E_WOU_TOO_BIG;
391+
}
392+
393+
uint64_t error_bound =
394+
state.error_bound_ns; // FIXME add sys clock error bound here
395+
double h_value = (double)state.holdover_multiplier_ns / FBCLOCK_POW2_16;
396+
397+
struct timespec ts;
398+
if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts) == -1) {
399+
return FBCLOCK_E_PTP_READ_OFFSET;
400+
}
401+
int64_t sysclock_time_now_ns =
402+
ts.tv_sec * NANOSECONDS_IN_SECONDS + ts.tv_nsec;
403+
404+
return fbclock_calculate_time_v2(
405+
error_bound,
406+
h_value,
407+
&state,
408+
state.phc_time_ns,
409+
state.sysclock_time_ns,
410+
sysclock_time_now_ns,
411+
truetime,
412+
time_standard);
272413
}
273414

274415
int fbclock_gettime(fbclock_lib* lib, fbclock_truetime* truetime) {
416+
if (lib->shmp_v2) {
417+
return fbclock_gettime_tz_v2(lib, truetime, FBCLOCK_TAI);
418+
}
275419
return fbclock_gettime_tz(lib, truetime, FBCLOCK_TAI);
276420
}
277421

278422
int fbclock_gettime_utc(fbclock_lib* lib, fbclock_truetime* truetime) {
423+
if (lib->shmp_v2) {
424+
return fbclock_gettime_tz_v2(lib, truetime, FBCLOCK_UTC);
425+
}
279426
return fbclock_gettime_tz(lib, truetime, FBCLOCK_UTC);
280427
}
281428

@@ -333,6 +480,42 @@ uint64_t fbclock_apply_utc_offset(
333480
multiplier);
334481
}
335482

483+
uint64_t fbclock_apply_utc_offset_v2(
484+
fbclock_clockdata_v2* state,
485+
int64_t phctime_ns) {
486+
// Fixed offset is applied if tzdata information not in shared memory
487+
if (state->utc_offset_pre_s == 0 && state->utc_offset_post_s == 0) {
488+
phctime_ns += UTC_TAI_OFFSET_NS;
489+
return (uint64_t)phctime_ns;
490+
}
491+
492+
fbclock_debug_print(
493+
"UTC-TAI Offset Before Leap Second Event: %d\n", state->utc_offset_pre_s);
494+
fbclock_debug_print(
495+
"UTC-TAI Offset After Leap Second Event: %d\n", state->utc_offset_post_s);
496+
fbclock_debug_print(
497+
"Clock Smearing Start Time (TAI): %lu\n", state->clock_smearing_start_s);
498+
fbclock_debug_print(
499+
"Clock Smearing End Time (TAI): %lu\n", state->clock_smearing_end_s);
500+
501+
// Multipler may be negative (if a negative leap second is applied)
502+
int multiplier = state->utc_offset_post_s - state->utc_offset_pre_s;
503+
504+
// Switch to nanoseconds
505+
uint64_t smear_end_ns = state->clock_smearing_end_s * 1e9;
506+
uint64_t smear_start_ns = state->clock_smearing_start_s * 1e9;
507+
uint64_t offset_post_ns = state->utc_offset_post_s * 1e9;
508+
uint64_t offset_pre_ns = state->utc_offset_pre_s * 1e9;
509+
510+
return fbclock_apply_smear(
511+
phctime_ns,
512+
offset_pre_ns,
513+
offset_post_ns,
514+
smear_start_ns,
515+
smear_end_ns,
516+
multiplier);
517+
}
518+
336519
const char* fbclock_strerror(int err_code) {
337520
const char* err_info = "unknown error";
338521
switch (err_code) {

0 commit comments

Comments
 (0)