Skip to content

Commit 8990348

Browse files
GUIGUI
committed
Improve resiliency of proxy during MongoDB replicaset changes.
If connecting to MongoDB that's part of a replicaset, these changes help improve the failover handling during replicaset changes (either planned or unplanned). The default read preference is now "primaryPreferred," which allows connections to secondaries during replicaset changes when a primary has not been elected yet. Mora was previously lacking this read preference functionality, so a pull request to Mora has been submitted to provide support. We're also handling more potential edge-cases that might crop up during replicaset changes, and retrying the queries in those cases. This better ensures no connections are dropped, even in the event of unexpected replicaset changes. The updates to the testing suite helped uncover these failover edge-cases, so these improvements should also resolve the possibility of sporadic test failures.
1 parent 7d63797 commit 8990348

14 files changed

Lines changed: 126 additions & 82 deletions

File tree

‎Gemfile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,5 +64,8 @@ gem "lazyhash", "~> 0.1.1"
6464
# Generating fake strings and data.
6565
gem "faker", "~> 1.6.6"
6666

67+
# Concurrency helpers.
68+
gem "concurrent-ruby", "~> 1.0.2"
69+
6770
# Debug printing
6871
gem "awesome_print", "~> 1.7.0"

‎Gemfile.lock‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ DEPENDENCIES
146146
capybara (~> 2.10.1)
147147
capybara-screenshot (~> 1.0.14)
148148
childprocess (~> 0.5.9)
149+
concurrent-ruby (~> 1.0.2)
149150
database_cleaner (~> 1.5.3)
150151
elasticsearch (~> 2.0.0)
151152
elasticsearch-persistence (~> 0.1.9)
@@ -168,4 +169,4 @@ DEPENDENCIES
168169
typhoeus (~> 1.1.0)
169170

170171
BUNDLED WITH
171-
1.13.1
172+
1.13.6

‎build/cmake/mora.cmake‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,17 @@ set(GLIDE_SOURCE_DIR ${SOURCE_DIR})
2626
ExternalProject_Add(
2727
mora
2828
DEPENDS glide
29-
URL https://github.com/emicklei/mora/archive/${MORA_VERSION}.tar.gz
29+
# Use fork for read preference support:
30+
# https://github.com/emicklei/mora/pull/44
31+
URL https://github.com/GUI/mora/archive/${MORA_VERSION}.tar.gz
3032
URL_HASH MD5=${MORA_HASH}
3133
SOURCE_DIR ${WORK_DIR}/gocode/src/github.com/emicklei/mora
3234
BUILD_IN_SOURCE 1
3335
CONFIGURE_COMMAND ""
3436
BUILD_COMMAND cp ${CMAKE_SOURCE_DIR}/build/mora/glide.yaml <SOURCE_DIR>/glide.yaml
3537
COMMAND cp ${CMAKE_SOURCE_DIR}/build/mora/glide.lock <SOURCE_DIR>/glide.lock
36-
COMMAND env PATH=${GOLANG_SOURCE_DIR}/bin:${GLIDE_SOURCE_DIR}:${WORK_DIR}/gocode/bin:$ENV{PATH} GOPATH=${WORK_DIR}/gocode GOROOT=${GOLANG_SOURCE_DIR} GO15VENDOREXPERIMENT=1 glide install
37-
COMMAND env PATH=${GOLANG_SOURCE_DIR}/bin:${GLIDE_SOURCE_DIR}:${WORK_DIR}/gocode/bin:$ENV{PATH} GOPATH=${WORK_DIR}/gocode GOROOT=${GOLANG_SOURCE_DIR} GO15VENDOREXPERIMENT=1 go install
38+
COMMAND env PATH=${GOLANG_SOURCE_DIR}/bin:${GLIDE_SOURCE_DIR}:${WORK_DIR}/gocode/bin:$ENV{PATH} GOPATH=${WORK_DIR}/gocode GOROOT=${GOLANG_SOURCE_DIR} glide install
39+
COMMAND env PATH=${GOLANG_SOURCE_DIR}/bin:${GLIDE_SOURCE_DIR}:${WORK_DIR}/gocode/bin:$ENV{PATH} GOPATH=${WORK_DIR}/gocode GOROOT=${GOLANG_SOURCE_DIR} go install
3840
INSTALL_COMMAND install -D -m 755 ${WORK_DIR}/gocode/bin/mora ${STAGE_EMBEDDED_DIR}/bin/mora
3941
)
4042
ExternalProject_Add_Step(

‎build/cmake/versions.cmake‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,8 @@ set(MAILHOG_HASH 6602fd7f69276e7efba310362e958133)
6868
set(MONGO_ORCHESTRATION_VERSION 0.6.7)
6969
set(MONGODB_VERSION 3.2.11)
7070
set(MONGODB_HASH 9916a076bd2e2fa8e8fbad94bb083fae)
71-
set(MORA_VERSION 4cae0b86a440356cc3b669fb76343ac514c99655)
72-
set(MORA_HASH 6764886ca9b8c5302e93597c4500bfd3)
71+
set(MORA_VERSION 02c69fb82839e4fc2c8415c763f1934b7cf7dd4f)
72+
set(MORA_HASH 8030e2869ac1e9b6ef90c770cd4e946f)
7373
set(NGX_DYUPS_VERSION d4b3e053dee10e2879882eb4c346ac7d534e2d14)
7474
set(NGX_DYUPS_HASH bdf4408599602afa38365a426e126d21)
7575
set(NGX_TXID_VERSION f1c197cb9c42e364a87fbb28d5508e486592ca42)

‎build/mora/glide.lock‎

Lines changed: 7 additions & 11 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎build/mora/glide.yaml‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package: .
1+
package: github.com/emicklei/mora
22
import:
33
- package: github.com/compose/mejson
44
- package: github.com/emicklei/go-restful
@@ -7,15 +7,9 @@ import:
77
version: ~1.1.3
88
subpackages:
99
- swagger
10-
- package: github.com/emicklei/mora
11-
subpackages:
12-
- api/documents
13-
- api/response
14-
- api/statistics
15-
- session
1610
- package: github.com/magiconair/properties
1711
version: ~1.7.0
1812
- package: gopkg.in/mgo.v2
19-
version: r2016.02.04
13+
version: r2016.08.01
2014
subpackages:
2115
- bson

‎build/scripts/outdated‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,9 @@ repos = {
118118
:constraint => "~> 3.2.9",
119119
},
120120
"mora" => {
121-
:git => "https://github.com/emicklei/mora.git",
121+
# Use fork for read preference support:
122+
# https://github.com/emicklei/mora/pull/44
123+
:git => "https://github.com/GUI/mora.git",
122124
:git_ref => "master",
123125
},
124126
"ngx_dyups" => {

‎config/default.yml‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -128,14 +128,7 @@ dns_resolver:
128128
retries: 3
129129
mongodb:
130130
url: "mongodb://127.0.0.1:14001/api_umbrella"
131-
options:
132-
server:
133-
auto_reconnect: true
134-
socketOptions:
135-
keepAlive: 500
136-
replset:
137-
socketOptions:
138-
keepAlive: 500
131+
read_preference: primaryPreferred
139132
embedded_server_config:
140133
processManagement:
141134
fork: false

‎src/api-umbrella/utils/mongo.lua‎

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
local cjson = require "cjson"
22
local http = require "resty.http"
3+
local stringx = require "pl.stringx"
34
local types = require "pl.types"
45

56
local is_empty = types.is_empty
7+
local startswith = stringx.startswith
68

79
local _M = {}
810

@@ -68,22 +70,39 @@ local function perform_query(path, query_options, http_options)
6870

6971
local response, err = try_query(path, http_options)
7072

71-
-- If we get an "EOF" error from Mora, this means our query occurred during
72-
-- the middle of a server or replicaset change. In this case, retry the
73-
-- request a couple more times.
73+
-- If we certain types of errors from Mora, this means our query occurred
74+
-- during the middle of a server or replicaset change. In this case, retry
75+
-- the request a few more times.
7476
--
7577
-- This should be less likely in mora since
7678
-- https://github.com/emicklei/mora/pull/29, but it's still possible for this
7779
-- to crop up if the socket gets closed sometime between the request starting
78-
-- and the query actually executing. After more research, this seems to be
80+
-- and the query actually executing. This can also happen in case of
81+
-- unexpected mongod shutdowns. After more research, this seems to be
7982
-- expected mgo behavior, and it's up to the app to handle these type of
8083
-- errors. I'm not entirely sure whether we should try to address the issue
8184
-- in mora itself, but in the meantime, we'll retry here.
82-
if err and err == "mongodb error: EOF" then
83-
response, err = try_query(path, http_options)
84-
if err and err == "mongodb error: EOF" then
85-
ngx.sleep(0.5)
86-
response, err = try_query(path, http_options)
85+
if err then
86+
-- Loop to retry a few times until no errors occurs or we give up, since we
87+
-- don't want to wait forever.
88+
local retries = 0
89+
while err and retries < 5 do
90+
if err == "mongodb error: EOF"
91+
or err == "mongodb error: node is recovering"
92+
or err == "mongodb error: interrupted at shutdown"
93+
or err == "mongodb error: Closed explicitly"
94+
or startswith(err, "mongodb error: read tcp")
95+
then
96+
-- Retry immediately, then sleep between further retries.
97+
retries = retries + 1
98+
if retries > 1 then
99+
ngx.sleep(0.5)
100+
end
101+
102+
response, err = try_query(path, http_options)
103+
else
104+
break
105+
end
87106
end
88107
end
89108

‎src/api-umbrella/web-app/config/application.rb‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,28 @@ class Application < Rails::Application
8787
require "js_locale_helper"
8888
end
8989

90+
# Instead of loading from a mongoid.yml file, load the Mongoid config in
91+
# code, where it's easier to merge settings from our API Umbrella
92+
# configuration.
93+
initializer "mongoid-config", :after => "mongoid.load-config" do
94+
config = {
95+
:clients => {
96+
:default => {
97+
:uri => ApiUmbrellaConfig[:mongodb][:url],
98+
:options => {
99+
:read => {
100+
:mode => ApiUmbrellaConfig[:mongodb][:read_preference].underscore.to_sym,
101+
},
102+
},
103+
},
104+
},
105+
}
106+
107+
Mongoid::Clients.disconnect
108+
Mongoid::Clients.clear
109+
Mongoid.load_configuration(config)
110+
end
111+
90112
# Settings in config/environments/* take precedence over those specified here.
91113
# Application configuration should go into files in config/initializers
92114
# -- all .rb files in that directory are automatically loaded.

0 commit comments

Comments
 (0)