Skip to content
This repository was archived by the owner on Dec 24, 2022. It is now read-only.

Commit fe35f5d

Browse files
mythzmythz
committed
Add new SentinelSentinels() API and reset Sentinels on Start and when Redis Master is objectively down
1 parent 02dd8c3 commit fe35f5d

File tree

6 files changed

+132
-30
lines changed

6 files changed

+132
-30
lines changed

‎src/ServiceStack.Redis/Commands.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,7 @@ public static class Commands
206206
// Sentinel commands
207207
public readonly static byte[] Sentinel = "SENTINEL".ToUtf8Bytes();
208208
public readonly static byte[] Masters = "masters".ToUtf8Bytes();
209+
public readonly static byte[] Sentinels = "sentinels".ToUtf8Bytes();
209210
public readonly static byte[] Master = "master".ToUtf8Bytes();
210211
public readonly static byte[] Slaves = "slaves".ToUtf8Bytes();
211212
public readonly static byte[] GetMasterAddrByName = "get-master-addr-by-name".ToUtf8Bytes();

‎src/ServiceStack.Redis/RedisNativeClient.cs‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,6 @@ public partial class RedisNativeClient
3535
{
3636
private static readonly ILog log = LogManager.GetLogger(typeof(RedisNativeClient));
3737

38-
public const long DefaultDb = 0;
39-
public const int DefaultPort = 6379;
40-
public const int DefaultPortSsl = 6380;
41-
public const int DefaultPortSentinel = 26379;
42-
public const string DefaultHost = "localhost";
43-
public const int DefaultIdleTimeOutSecs = 240; //default on redis is 300
44-
4538
internal const int Success = 1;
4639
internal const int OneGb = 1073741824;
4740
private readonly byte[] endData = new[] { (byte)'\r', (byte)'\n' };
@@ -143,7 +136,7 @@ public RedisNativeClient(RedisEndpoint config)
143136
public RedisNativeClient(string host, int port)
144137
: this(host, port, null) { }
145138

146-
public RedisNativeClient(string host, int port, string password = null, long db = DefaultDb)
139+
public RedisNativeClient(string host, int port, string password = null, long db = RedisConfig.DefaultDb)
147140
{
148141
if (host == null)
149142
throw new ArgumentNullException("host");
@@ -168,7 +161,7 @@ private void Init(RedisEndpoint config)
168161
}
169162

170163
public RedisNativeClient()
171-
: this(DefaultHost, DefaultPort) { }
164+
: this(RedisConfig.DefaultHost, RedisConfig.DefaultPort) { }
172165

173166
#region Common Operations
174167

@@ -1451,7 +1444,7 @@ public byte[] BRPopLPush(string fromListId, string toListId, int timeOutSecs)
14511444

14521445
private static Dictionary<string, string> ToDictionary(object[] result)
14531446
{
1454-
var masterInfo = new Dictionary<string, string>();
1447+
var map = new Dictionary<string, string>();
14551448

14561449
string key = null;
14571450
for (var i = 0; i < result.Length; i++)
@@ -1464,10 +1457,10 @@ private static Dictionary<string, string> ToDictionary(object[] result)
14641457
else
14651458
{
14661459
var val = bytes.FromUtf8Bytes();
1467-
masterInfo[key] = val;
1460+
map[key] = val;
14681461
}
14691462
}
1470-
return masterInfo;
1463+
return map;
14711464
}
14721465

14731466
public List<Dictionary<string, string>> SentinelMasters()
@@ -1493,6 +1486,18 @@ public Dictionary<string, string> SentinelMaster(string masterName)
14931486
return ToDictionary(results);
14941487
}
14951488

1489+
public List<Dictionary<string, string>> SentinelSentinels(string masterName)
1490+
{
1491+
var args = new List<byte[]>
1492+
{
1493+
Commands.Sentinel,
1494+
Commands.Sentinels,
1495+
masterName.ToUtf8Bytes(),
1496+
};
1497+
var results = SendExpectDeeplyNestedMultiData(args.ToArray());
1498+
return (from object[] result in results select ToDictionary(result)).ToList();
1499+
}
1500+
14961501
public List<Dictionary<string, string>> SentinelSlaves(string masterName)
14971502
{
14981503
var args = new List<byte[]>

‎src/ServiceStack.Redis/RedisSentinel.cs‎

Lines changed: 71 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@ public string MasterName
3333

3434
private int failures = 0;
3535
private int sentinelIndex = -1;
36-
internal RedisEndpoint[] SentinelHosts { get; private set; }
36+
public List<string> SentinelHosts { get; private set; }
37+
internal RedisEndpoint[] SentinelEndpoints { get; private set; }
3738
private RedisSentinelWorker worker;
3839
private static int MaxFailures = 5;
3940

@@ -44,30 +45,34 @@ public string MasterName
4445

4546
public Dictionary<string, string> IpAddressMap { get; set; }
4647

48+
public bool ScanForOtherSentinels { get; set; }
4749
public TimeSpan WaitBetweenSentinelLookups { get; set; }
4850
public TimeSpan MaxWaitBetweenSentinelLookups { get; set; }
4951
public int SentinelWorkerTimeoutMs { get; set; }
5052

5153
public bool ResetWhenSubjectivelyDown { get; set; }
5254
public bool ResetWhenObjectivelyDown { get; set; }
55+
public bool ResetSentinelsWhenObjectivelyDown { get; set; }
5356

5457
public RedisSentinel(string sentinelHost = null, string masterName = null)
5558
: this(new[] { sentinelHost ?? DefaultAddress }, masterName ?? DefaultMasterName) { }
5659

5760
public RedisSentinel(IEnumerable<string> sentinelHosts, string masterName = null)
5861
{
5962
this.SentinelHosts = sentinelHosts != null
60-
? sentinelHosts.Map(x => x.ToRedisEndpoint(defaultPort:RedisNativeClient.DefaultPortSentinel)).ToArray()
63+
? sentinelHosts.ToList()
6164
: null;
6265

63-
if (SentinelHosts == null || SentinelHosts.Length == 0)
66+
if (SentinelHosts == null || SentinelHosts.Count == 0)
6467
throw new ArgumentException("sentinels must have at least one entry");
6568

6669
this.masterName = masterName ?? DefaultMasterName;
6770
IpAddressMap = new Dictionary<string, string>();
6871
RedisManagerFactory = (masters,slaves) => new PooledRedisClientManager(masters, slaves);
72+
ScanForOtherSentinels = true;
6973
ResetWhenObjectivelyDown = true;
7074
ResetWhenSubjectivelyDown = true;
75+
ResetSentinelsWhenObjectivelyDown = true;
7176
SentinelWorkerTimeoutMs = 100;
7277
WaitBetweenSentinelLookups = TimeSpan.FromMilliseconds(250);
7378
MaxWaitBetweenSentinelLookups = TimeSpan.FromSeconds(60);
@@ -80,15 +85,69 @@ public IRedisClientsManager Start()
8085
{
8186
lock (oLock)
8287
{
83-
GetValidSentinel();
88+
if (ScanForOtherSentinels)
89+
{
90+
var activeHosts = GetActiveSentinelHosts(SentinelHosts);
91+
if (activeHosts.Count == 0)
92+
throw new ArgumentException("Could not find any active sentinels from: ",
93+
string.Join(", ", SentinelHosts.ToArray()));
94+
95+
SentinelHosts = activeHosts;
96+
}
8497

85-
if (this.RedisManager == null)
98+
SentinelEndpoints = SentinelHosts
99+
.Map(x => x.ToRedisEndpoint(defaultPort:RedisConfig.DefaultPortSentinel))
100+
.ToArray();
101+
102+
var sentinelWorker = GetValidSentinelWorker();
103+
if (this.RedisManager == null || sentinelWorker == null)
86104
throw new ApplicationException("Unable to resolve sentinels!");
87105

88106
return this.RedisManager;
89107
}
90108
}
91109

110+
public List<string> GetActiveSentinelHosts(IEnumerable<string> sentinelHosts)
111+
{
112+
var activeSentinelHosts = new List<string>();
113+
foreach (var sentinelHost in sentinelHosts.ToArray())
114+
{
115+
try
116+
{
117+
var endpoint = sentinelHost.ToRedisEndpoint(defaultPort: RedisConfig.DefaultPortSentinel);
118+
using (var sentinelWorker = new RedisSentinelWorker(this, endpoint))
119+
{
120+
var activeHosts = sentinelWorker.GetSentinelHosts(MasterName);
121+
122+
if (!activeSentinelHosts.Contains(sentinelHost))
123+
activeSentinelHosts.Add(sentinelHost);
124+
125+
foreach (var activeHost in activeHosts)
126+
{
127+
if (!activeSentinelHosts.Contains(activeHost))
128+
activeSentinelHosts.Add(activeHost);
129+
}
130+
}
131+
}
132+
catch (Exception ex)
133+
{
134+
Log.Error("Could not get active Sentinels from: {0}".Fmt(sentinelHost), ex);
135+
}
136+
}
137+
return activeSentinelHosts;
138+
}
139+
140+
public void ResetSentinels()
141+
{
142+
var activeHosts = GetActiveSentinelHosts(SentinelHosts);
143+
if (activeHosts.Count == 0) return;
144+
145+
SentinelHosts = activeHosts;
146+
SentinelEndpoints = SentinelHosts
147+
.Map(x => x.ToRedisEndpoint(defaultPort: RedisConfig.DefaultPortSentinel))
148+
.ToArray();
149+
}
150+
92151
public Func<string, string> HostFilter { get; set; }
93152

94153
internal string[] ConfigureHosts(IEnumerable<string> hosts)
@@ -143,7 +202,7 @@ public IRedisClientsManager GetRedisManager()
143202
return RedisManager ?? (RedisManager = CreateRedisManager(GetSentinelInfo()));
144203
}
145204

146-
private RedisSentinelWorker GetValidSentinel()
205+
private RedisSentinelWorker GetValidSentinelWorker()
147206
{
148207
if (this.worker != null)
149208
return this.worker;
@@ -178,7 +237,7 @@ private RedisSentinelWorker GetValidSentinel()
178237

179238
public RedisEndpoint GetMaster()
180239
{
181-
var sentinelWorker = GetValidSentinel();
240+
var sentinelWorker = GetValidSentinelWorker();
182241
lock (sentinelWorker)
183242
{
184243
var host = sentinelWorker.GetMasterHost(masterName);
@@ -190,7 +249,7 @@ public RedisEndpoint GetMaster()
190249

191250
public List<RedisEndpoint> GetSlaves()
192251
{
193-
var sentinelWorker = GetValidSentinel();
252+
var sentinelWorker = GetValidSentinelWorker();
194253
lock (sentinelWorker)
195254
{
196255
var hosts = sentinelWorker.GetSlaveHosts(masterName);
@@ -205,7 +264,7 @@ public List<RedisEndpoint> GetSlaves()
205264
/// <remarks>This will be true if the failures is less than either RedisSentinel.MaxFailures or the # of sentinels, whatever is greater</remarks>
206265
private bool ShouldRetry()
207266
{
208-
return this.failures < Math.Max(MaxFailures, this.SentinelHosts.Length);
267+
return this.failures < Math.Max(MaxFailures, this.SentinelEndpoints.Length);
209268
}
210269

211270
private RedisSentinelWorker GetNextSentinel()
@@ -220,10 +279,10 @@ private RedisSentinelWorker GetNextSentinel()
220279
this.worker = null;
221280
}
222281

223-
if (sentinelIndex >= SentinelHosts.Length)
282+
if (sentinelIndex >= SentinelEndpoints.Length)
224283
sentinelIndex = 0;
225284

226-
var sentinelWorker = new RedisSentinelWorker(this, SentinelHosts[sentinelIndex])
285+
var sentinelWorker = new RedisSentinelWorker(this, SentinelEndpoints[sentinelIndex])
227286
{
228287
OnSentinelError = OnSentinelError
229288
};
@@ -248,7 +307,7 @@ private void OnSentinelError(Exception ex)
248307

249308
public SentinelInfo GetSentinelInfo()
250309
{
251-
var sentinelWorker = GetValidSentinel();
310+
var sentinelWorker = GetValidSentinelWorker();
252311
lock (sentinelWorker)
253312
{
254313
return sentinelWorker.GetSentinelInfo();

‎src/ServiceStack.Redis/RedisSentinelWorker.cs‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,11 @@ private void SentinelMessageReceived(string channel, string message)
4747
if (isObjectivelyDown)
4848
Interlocked.Increment(ref RedisState.TotalObjectiveServersDown);
4949

50+
if (isObjectivelyDown && sentinel.ResetSentinelsWhenObjectivelyDown)
51+
{
52+
sentinel.ResetSentinels();
53+
}
54+
5055
if (c == "+failover-end"
5156
|| (sentinel.ResetWhenSubjectivelyDown && isSubjectivelyDown)
5257
|| (sentinel.ResetWhenObjectivelyDown && isObjectivelyDown))
@@ -86,12 +91,17 @@ internal string GetMasterHost(string masterName)
8691
return null;
8792
}
8893

94+
internal List<string> GetSentinelHosts(string masterName)
95+
{
96+
return SanitizeHostsConfig(this.sentinelClient.SentinelSentinels(sentinel.MasterName));
97+
}
98+
8999
internal List<string> GetSlaveHosts(string masterName)
90100
{
91-
return SanitizeSlavesConfig(this.sentinelClient.SentinelSlaves(sentinel.MasterName));
101+
return SanitizeHostsConfig(this.sentinelClient.SentinelSlaves(sentinel.MasterName));
92102
}
93103

94-
private List<string> SanitizeSlavesConfig(IEnumerable<Dictionary<string, string>> slaves)
104+
private List<string> SanitizeHostsConfig(IEnumerable<Dictionary<string, string>> slaves)
95105
{
96106
string ip;
97107
string port;
@@ -125,7 +135,7 @@ public void BeginListeningForConfigurationChanges()
125135
var sentinelManager = new BasicRedisClientManager(sentinel.SentinelHosts, sentinel.SentinelHosts)
126136
{
127137
//Use BasicRedisResolver which doesn't validate non-Master Sentinel instances
128-
RedisResolver = new BasicRedisResolver(sentinel.SentinelHosts, sentinel.SentinelHosts)
138+
RedisResolver = new BasicRedisResolver(sentinel.SentinelEndpoints, sentinel.SentinelEndpoints)
129139
};
130140
this.sentinePubSub = new RedisPubSubServer(sentinelManager)
131141
{

‎tests/ServiceStack.Redis.Tests/PooledRedisClientManagerTests.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ public void Can_get_ReadWrite_client()
177177
private static void AssertClientHasHost(IRedisClient client, string hostWithOptionalPort)
178178
{
179179
var parts = hostWithOptionalPort.Split(':');
180-
var port = parts.Length > 1 ? int.Parse(parts[1]) : RedisNativeClient.DefaultPort;
180+
var port = parts.Length > 1 ? int.Parse(parts[1]) : RedisConfig.DefaultPort;
181181

182182
Assert.That(client.Host, Is.EqualTo(parts[0]));
183183
Assert.That(client.Port, Is.EqualTo(port));

‎tests/ServiceStack.Redis.Tests/RedisSentinelTests.cs‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,15 @@ public void Can_Get_Sentinel_Slaves()
7777
Assert.That(slaves.Count, Is.GreaterThan(0));
7878
}
7979

80+
[Test]
81+
public void Can_Get_Sentinel_Sentinels()
82+
{
83+
var sentinels = RedisSentinel.SentinelSentinels(MasterName);
84+
sentinels.PrintDump();
85+
86+
Assert.That(sentinels.Count, Is.GreaterThan(0));
87+
}
88+
8089
[Test]
8190
public void Can_Get_Master_Addr()
8291
{
@@ -90,6 +99,24 @@ public void Can_Get_Master_Addr()
9099
Assert.That(hostString, Is.EqualTo(MasterHosts[0]));
91100
}
92101

102+
[Test]
103+
public void Does_scan_for_other_active_sentinels()
104+
{
105+
using (var sentinel = new RedisSentinel(SentinelHosts[0]) {
106+
ScanForOtherSentinels = true
107+
})
108+
{
109+
var clientsManager = sentinel.Start();
110+
111+
Assert.That(sentinel.SentinelHosts, Is.EquivalentTo(SentinelHosts));
112+
113+
using (var client = clientsManager.GetClient())
114+
{
115+
Assert.That(client.GetHostString(), Is.EqualTo(MasterHosts[0]));
116+
}
117+
}
118+
}
119+
93120
[Test]
94121
public void Can_Get_Redis_ClientsManager()
95122
{
@@ -181,8 +208,8 @@ public void Run_sentinel_for_10_minutes()
181208
[Test]
182209
public void Defaults_to_default_sentinel_port()
183210
{
184-
var sentinelEndpoint = "127.0.0.1".ToRedisEndpoint(defaultPort: RedisNativeClient.DefaultPortSentinel);
185-
Assert.That(sentinelEndpoint.Port, Is.EqualTo(RedisNativeClient.DefaultPortSentinel));
211+
var sentinelEndpoint = "127.0.0.1".ToRedisEndpoint(defaultPort: RedisConfig.DefaultPortSentinel);
212+
Assert.That(sentinelEndpoint.Port, Is.EqualTo(RedisConfig.DefaultPortSentinel));
186213
}
187214
}
188215
}

0 commit comments

Comments
 (0)