Skip to content

Commit 3256d67

Browse files
cammelloscammellos
committed
Add clock values for public group/chats
I have extended and modified the current algorithm for message ordering so that it applies for group and public chats alike. We use Lamport timestamps but we prefix the unix timestamp, which should maximize the chances of the message being seen on the top of the chat. Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
1 parent b2f5146 commit 3256d67

17 files changed

Lines changed: 223 additions & 114 deletions

File tree

‎src/status_im/chat/console.cljs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
(:require [status-im.ui.components.styles :refer [default-chat-color]]
33
[status-im.utils.random :as random]
44
[status-im.constants :as constants]
5+
[status-im.utils.clocks :as utils.clocks]
56
[status-im.i18n :as i18n]
67
[clojure.string :as string]))
78

@@ -12,6 +13,7 @@
1213
:chat-id constants/console-chat-id
1314
:from constants/console-chat-id
1415
:to "me"
16+
:clock-value (utils.clocks/send 0)
1517
:content content
1618
:content-type content-type})
1719

@@ -27,8 +29,7 @@
2729
:contacts [{:identity constants/console-chat-id
2830
:text-color "#FFFFFF"
2931
:background-color "#AB7967"}]
30-
:last-to-clock-value 0
31-
:last-from-clock-value 0})
32+
:last-clock-value 0})
3233

3334
(def contact
3435
{:whisper-identity constants/console-chat-id

‎src/status_im/chat/events/receive_message.cljs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
[status-im.chat.events.commands :as commands-events]
55
[status-im.chat.models.message :as message-model]
66
[status-im.constants :as constants]
7+
[status-im.utils.clocks :as utils.clocks]
78
[status-im.utils.handlers :as handlers]
89
[status-im.utils.random :as random]))
910

@@ -65,6 +66,7 @@
6566
:content (str type ": " message)
6667
:content-type constants/content-type-log-message
6768
:outgoing false
69+
:clock-value (utils.clocks/send 0)
6870
:chat-id chat-id
6971
:from chat-id
7072
:to "me"}]))))
@@ -74,6 +76,7 @@
7476
:content (str content)
7577
:content-type constants/text-content-type
7678
:outgoing false
79+
:clock-value (utils.clocks/send 0)
7780
:chat-id chat-id
7881
:from chat-id
7982
:to "me"}])))))

‎src/status_im/chat/models.cljs‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,7 @@
2424
:is-active true
2525
:timestamp now
2626
:contacts [{:identity chat-id}]
27-
:last-from-clock-value 0
28-
:last-to-clock-value 0}))
27+
:last-clock-value 0}))
2928

3029
(defn add-chat
3130
"Adds new chat to db & realm, if the chat with same id already exists, justs restores it"
@@ -52,8 +51,7 @@
5251
:public? true
5352
:is-active true
5453
:timestamp now
55-
:last-to-clock-value 0
56-
:last-from-clock-value 0}]
54+
:last-clock-value 0}]
5755
{:db (assoc-in db [:chats topic] chat)
5856
:data-store/save-chat chat}))
5957

@@ -68,8 +66,7 @@
6866
:is-active true
6967
:timestamp now
7068
:contacts (mapv (partial hash-map :identity) participants)
71-
:last-to-clock-value 0
72-
:last-from-clock-value 0}]
69+
:last-clock-value 0}]
7370
{:db (assoc-in db [:chats chat-id] chat)
7471
:data-store/save-chat chat}))
7572

‎src/status_im/chat/models/message.cljs‎

Lines changed: 19 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
[status-im.chat.events.requests :as requests-events]
77
[status-im.chat.models :as chat-model]
88
[status-im.chat.models.commands :as commands-model]
9-
[status-im.utils.clocks :as clocks-utils]
9+
[status-im.utils.clocks :as utils.clocks]
1010
[status-im.utils.handlers :as handlers]
1111
[status-im.transport.utils :as transport.utils]
1212
[status-im.transport.message.core :as transport]
@@ -26,31 +26,18 @@
2626
(:ref (get available-commands-responses response-name))))
2727

2828
(defn- add-message
29-
[chat-id {:keys [message-id from-clock-value to-clock-value] :as message} current-chat? {:keys [db]}]
29+
[chat-id {:keys [message-id clock-value] :as message} current-chat? {:keys [db]}]
3030
(let [prepared-message (cond-> (assoc message :appearing? true)
3131
(not current-chat?)
3232
(assoc :appearing? false))]
33-
{:db (cond-> (-> db
34-
(update-in [:chats chat-id :messages] dissoc from-clock-value)
35-
(update-in [:chats chat-id :messages] assoc message-id prepared-message)
36-
(update-in [:chats chat-id :last-from-clock-value] max from-clock-value)
37-
(update-in [:chats chat-id :last-to-clock-value] max to-clock-value))
33+
{:db (cond->
34+
(-> db
35+
(update-in [:chats chat-id :messages] assoc message-id prepared-message)
36+
(update-in [:chats chat-id :last-clock-value] (partial utils.clocks/receive clock-value))) ; this will increase last-clock-value twice when sending our own messages
3837
(not current-chat?)
3938
(update-in [:chats chat-id :unviewed-messages] (fnil conj #{}) message-id))
4039
:data-store/save-message prepared-message}))
4140

42-
;; We start with [0 0] ([from-clock-value to-clock-value]) for each participant of 1-1 chat (local perspective on each device).
43-
;; Now for sending, we always increment the to-clock-value and include it in message payload being sent (so only to-clock-value is present in network message).
44-
;; Locally, the sent message always replicates the latest-from-clock-value of the chat.
45-
;; Upon receiving message, receiver reads the to-clock-value of the received message and sets that to be the from-clock-value locally
46-
;; (this will be also the new latest-from-clock-value of the chat), to-clock-value for the message is the latest-to-clock-value of the 1-1 chat`.
47-
48-
;; All this ensures, that there will be no [from-clock-value to-clock-value] duplicate in chat message on each device + the local order will appear consistent,
49-
;; even if it’s possible it won’t be the same on both devices (for example user A sends 5 messages, during the sending,
50-
;; he receives the message from user B, so his local order will be A1, A2, B, A3, A4, A5, but his messages will take a long time to reach user B,
51-
;; for some reason, so user B will see it as B, A1, A2, A3, A4, A5).
52-
;; I don’t think that’s very problematic and I don’t think we can do much about it without single source of truth where order received messages are serialised
53-
;; and definite order is established (server), it is the case even in the current implementation.
5441
(defn- prepare-chat [chat-id {:keys [db] :as cofx}]
5542
(if (get-in db [:chats chat-id])
5643
(chat-model/upsert-chat {:chat-id chat-id} cofx)
@@ -63,51 +50,31 @@
6350
(when send-seen?
6451
(transport/send (protocol/map->MessagesSeen {:message-ids #{message-id}}) chat-id cofx)))
6552

66-
(defn- placeholder-message [chat-id from timestamp temp-id to-clock]
67-
{:message-id temp-id
68-
:outgoing false
69-
:chat-id chat-id
70-
:from from
71-
:to "me"
72-
:content "Waiting for message to arrive..."
73-
:content-type constants/content-type-placeholder
74-
:show? true
75-
:from-clock-value temp-id
76-
:to-clock-value to-clock
77-
:timestamp timestamp})
78-
79-
(defn- add-placeholder-messages [chat-id from timestamp old-from-clock to-clock new-from-clock {:keys [db]}]
80-
(when (> (- new-from-clock old-from-clock) 1)
81-
{:db (reduce (fn [db temp-id]
82-
(assoc-in db [:chats chat-id :messages temp-id] (placeholder-message chat-id from timestamp temp-id to-clock)))
83-
db
84-
(range (inc old-from-clock) new-from-clock))}))
8553

8654
(defn- add-received-message
87-
[{:keys [from message-id chat-id content content-type timestamp to-clock-value] :as message}
55+
[{:keys [from message-id chat-id content content-type timestamp clock-value to-clock-value] :as message}
8856
{:keys [db now] :as cofx}]
8957
(let [{:keys [current-chat-id
9058
view-id
9159
access-scope->commands-responses]
9260
:contacts/keys [contacts]} db
9361
{:keys [public-key] :as current-account} (get-current-account db)
9462
current-chat? (and (= :chat view-id) (= current-chat-id chat-id))
95-
{:keys [last-from-clock-value
96-
last-to-clock-value] :as chat} (get-in db [:chats chat-id])
63+
{:keys [last-clock-value] :as chat} (get-in db [:chats chat-id])
9764
request-command (:request-command content)
9865
command-request? (and (= content-type constants/content-type-command-request)
9966
request-command)
100-
new-from-clock-value (or to-clock-value (inc last-from-clock-value))
10167
new-timestamp (or timestamp now)]
10268
(handlers/merge-fx cofx
10369
(add-message chat-id
10470
(cond-> (assoc message
10571
:timestamp new-timestamp
106-
:show? true
107-
:from-clock-value new-from-clock-value
108-
:to-clock-value last-to-clock-value)
72+
:show? true)
10973
public-key
11074
(assoc :user-statuses {public-key (if current-chat? :seen :received)})
75+
76+
(not clock-value)
77+
(assoc :clock-value (utils.clocks/send last-clock-value)) ; TODO (cammeelos): for backward compatibility, we use received time to be removed when not an issue anymore
11178
command-request?
11279
(assoc-in [:content :request-command-ref]
11380
(lookup-response-ref access-scope->commands-responses
@@ -116,8 +83,7 @@
11683
(send-message-seen chat-id message-id (and public-key
11784
current-chat?
11885
(not (chat-model/bot-only-chat? db chat-id))
119-
(not (= constants/system from))))
120-
(add-placeholder-messages chat-id from new-timestamp last-from-clock-value last-to-clock-value new-from-clock-value))))
86+
(not (= constants/system from)))))))
12187

12288
(defn receive
12389
[{:keys [chat-id message-id] :as message} cofx]
@@ -213,26 +179,25 @@
213179
(defn add-message-type [message {:keys [chat-id group-chat public?]}]
214180
(cond-> message
215181
(not group-chat)
216-
(assoc :message-type :user-message)
182+
(assoc :message-type :user-message)
217183
(and group-chat public?)
218184
(assoc :message-type :public-group-user-message)
219185
(and group-chat (not public?))
220186
(assoc :message-type :group-user-message)))
221187

222188
(defn- prepare-plain-message [chat-id {:keys [identity message-text]}
223-
{:keys [last-to-clock-value last-from-clock-value] :as chat} now]
189+
{:keys [last-clock-value] :as chat} now]
224190
(add-message-type {:chat-id chat-id
225191
:content message-text
226192
:from identity
227193
:content-type constants/text-content-type
228194
:outgoing true
229195
:timestamp now
230-
:to-clock-value (inc last-to-clock-value)
231-
:from-clock-value last-from-clock-value
196+
:clock-value (utils.clocks/send last-clock-value)
232197
:show? true}
233198
chat))
234199

235-
(def ^:private transport-keys [:content :content-type :message-type :to-clock-value :timestamp])
200+
(def ^:private transport-keys [:content :content-type :message-type :clock-value :timestamp])
236201

237202
(defn- upsert-and-send [{:keys [chat-id] :as message} cofx]
238203
(let [send-record (protocol/map->Message (select-keys message transport-keys))
@@ -247,7 +212,7 @@
247212

248213
(defn- prepare-command-message
249214
[identity
250-
{:keys [last-to-clock-value last-from-clock-value chat-id] :as chat}
215+
{:keys [last-clock-value chat-id] :as chat}
251216
now
252217
{request-params :params
253218
request-command :command
@@ -280,8 +245,7 @@
280245
constants/content-type-command-request
281246
constants/content-type-command))
282247
:outgoing true
283-
:to-clock-value (inc last-to-clock-value)
284-
:from-clock-value last-from-clock-value
248+
:clock-value (utils.clocks/send last-clock-value)
285249
:show? true}
286250
chat)))
287251

‎src/status_im/chat/subs.cljs‎

Lines changed: 39 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -91,25 +91,48 @@
9191
(fn [[chats id] [_ k chat-id]]
9292
(get-in chats [(or chat-id id) k])))
9393

94+
(defn- partition-by-datemark
95+
"Reduce step which expects the input list of messages to be sorted by clock value.
96+
It makes best effort to group them by day.
97+
We cannot sort them by :timestamp, as that represents the clock of the sender
98+
and we have no guarantees on the order.
99+
100+
We naively and arbitrarly group them assuming that out-of-order timestamps
101+
fall in the previous bucket.
102+
103+
A sends M1 to B with timestamp 2000-01-01T00:00:00
104+
B replies M2 with timestamp 1999-12-31-23:59:59
105+
106+
M1 needs to be displayed before M2
107+
108+
so we bucket both in 1999-12-31"
109+
[{:keys [acc last-timestamp last-datemark]} {:keys [timestamp datemark] :as msg}]
110+
(if (or (empty? acc) ; initial element
111+
(and (not= last-datemark datemark) ; not the same day
112+
(< timestamp last-timestamp))) ; not out-of-order
113+
{:last-timestamp timestamp
114+
:last-datemark datemark
115+
:acc (conj acc [datemark [msg]])} ; add new datemark group
116+
{:last-timestamp (max timestamp last-timestamp)
117+
:last-datemark last-datemark
118+
:acc (conj (pop acc) (update (peek acc) 1 conj msg))})) ; conj to the last element
119+
94120
(defn message-datemark-groups
95121
"Transforms map of messages into sequence of `[datemark messages]` tuples, where
96-
messages with particular datemark are sorted according to their `:clock-values` and
97-
tuples themeselves are sorted according to the highest `:clock-values` in the messages."
122+
messages with particular datemark are sorted according to their clock-values."
98123
[id->messages]
99-
(let [clock-sorter (juxt :from-clock-value :to-clock-value)
100-
datemark->messages (transduce (comp (map second)
101-
(filter :show?)
102-
(map (fn [{:keys [timestamp] :as msg}]
103-
(assoc msg :datemark (time/day-relative timestamp)))))
104-
(completing (fn [acc {:keys [datemark] :as msg}]
105-
(update acc datemark conj msg)))
106-
{}
107-
id->messages)]
108-
(->> datemark->messages
109-
(map (fn [[datemark messages]]
110-
[datemark (->> messages (sort-by clock-sorter) reverse)]))
111-
(sort-by (comp clock-sorter first second))
112-
reverse)))
124+
(let [sorted-messages (->> id->messages
125+
vals
126+
(sort-by (juxt (comp unchecked-negate :clock-value) :message-id))) ; sort-by clock in reverse order, break ties by :message-id field
127+
remove-hidden-xf (filter :show?)
128+
add-datemark-xf (map (fn [{:keys [timestamp] :as msg}]
129+
(assoc msg :datemark (time/day-relative timestamp))))]
130+
(-> (transduce (comp remove-hidden-xf
131+
add-datemark-xf)
132+
(completing partition-by-datemark)
133+
{:acc []}
134+
sorted-messages)
135+
:acc)))
113136

114137
(reg-sub
115138
:get-chat-message-datemark-groups

‎src/status_im/data_store/messages.cljs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@
9090
(data-store/save (prepare-message (merge default-values
9191
message
9292
{:from (or from "anonymous")
93-
:timestamp (datetime/timestamp)})))))
93+
:received-timestamp (datetime/timestamp)})))))
9494

9595
(re-frame/reg-fx
9696
:data-store/save-message

‎src/status_im/data_store/realm/chats.cljs‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,10 @@
77
(:refer-clojure :exclude [exists?]))
88

99
(defn- normalize-chat [{:keys [chat-id] :as chat}]
10-
(let [last-to-clock-value (messages/get-last-clock-value chat-id :to-clock-value)
11-
last-from-clock-value (messages/get-last-clock-value chat-id :from-clock-value)]
10+
(let [last-clock-value (messages/get-last-clock-value chat-id)]
1211
(-> chat
1312
(realm/fix-map->vec :contacts)
14-
(merge {:last-to-clock-value (or last-to-clock-value 0)
15-
:last-from-clock-value (or last-from-clock-value 0)}))))
13+
(assoc :last-clock-value (or last-clock-value 0)))))
1614

1715
(defn get-all-active
1816
[]

‎src/status_im/data_store/realm/messages.cljs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,11 +59,11 @@
5959
realm/js-object->clj))
6060

6161
(defn get-last-clock-value
62-
[chat-id clock-prop]
62+
[chat-id]
6363
(-> (realm/get-by-field @realm/account-realm :message :chat-id chat-id)
64-
(realm/sorted clock-prop :desc)
64+
(realm/sorted :clock-value :desc)
6565
(realm/single-clj)
66-
(get clock-prop)))
66+
:clock-value))
6767

6868
(defn get-unviewed
6969
[current-public-key]

‎src/status_im/data_store/realm/schemas/account/core.cljs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,4 @@
77
(def schemas [{:schema v1/schema
88
:schemaVersion 1
99
:migration v1/migration}])
10+

‎src/status_im/data_store/realm/schemas/account/v1/message.cljs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,7 @@
2222
:optional true}
2323
:user-statuses {:type :list
2424
:objectType :user-status}
25-
:from-clock-value {:type :int
26-
:default 0}
27-
:to-clock-value {:type :int
25+
:clock-value {:type :int
2826
:default 0}
2927
:show? {:type :bool
3028
:default true}}})

0 commit comments

Comments
 (0)