Skip to content

Commit 4ac0179

Browse files
jimjagjimjag
committed
Merge r420936, r420954, r420986, r421283 from trunk:
Restructure the find_route_worker() function. Basically, it should take care of knowing about usable and unusable workers. By centralizing this logic, it will make it easier and more streamlined to add in the forthcoming hot-standby status workers. Clean up some proxy macros. Avoid the use of magic numbers, and instead use pre-defined defines. Also, ensure that usable workers have been initialized :) Allocate a bit for hot standbys. Adjust so that normal "usable" workers don't count these. Add in hot-standby balancer member. If all other members are disabled or not-usable, ONLY THEN will the hot standby's be used. Reviewed by: jim git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/branches/2.2.x@438563 13f79535-47bb-0310-9956-ffa450edef68
1 parent 698c010 commit 4ac0179

6 files changed

Lines changed: 149 additions & 104 deletions

File tree

‎CHANGES‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
-*- coding: utf-8 -*-
22
Changes with Apache 2.2.4
33

4+
*) mod_proxy_balancer: Workers can now be defined as "hot standby" which
5+
will only be used if all other workers are unusable (eg: in
6+
error or disabled). Also, the balancer-manager displays the election
7+
count and I/O counts of all workers. [Jim Jagielski]
8+
49
*) mod_proxy_ajp: Close connection to backend if reading of request body
510
fails. PR 40310. [Ian Abel <ianabel mxtelecom.com>]
611

‎STATUS‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -94,16 +94,6 @@ PATCHES ACCEPTED TO BACKPORT FROM TRUNK:
9494
Trunk version works
9595
+1: jim, mturk, rpluem
9696

97-
* mod_proxy_balancer: Add in Hot Standby worker functionality.
98-
Trunk version of patch:
99-
http://svn.apache.org/viewvc?view=rev&revision=420936
100-
http://svn.apache.org/viewvc?view=rev&revision=420954
101-
http://svn.apache.org/viewvc?view=rev&revision=420986
102-
http://svn.apache.org/viewvc?view=rev&revision=421283
103-
2.2.x version of patch:
104-
http://people.apache.org/~jim/patches/hot-standby.txt
105-
+1: jim, mturk, rpluem
106-
10797
PATCHES PROPOSED TO BACKPORT FROM TRUNK:
10898

10999
* mpm_winnt: Fix return values from wait_for_many_objects.

‎modules/proxy/mod_proxy.c‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -213,8 +213,14 @@ static const char *set_worker_param(apr_pool_t *p,
213213
else
214214
worker->status &= ~PROXY_WORKER_IN_ERROR;
215215
}
216+
else if (*v == 'H' || *v == 'h') {
217+
if (mode)
218+
worker->status |= PROXY_WORKER_HOT_STANDBY;
219+
else
220+
worker->status &= ~PROXY_WORKER_HOT_STANDBY;
221+
}
216222
else {
217-
return "Unknow status parameter option";
223+
return "Unknown status parameter option";
218224
}
219225
}
220226
}

‎modules/proxy/mod_proxy.h‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -241,15 +241,26 @@ struct proxy_conn_pool {
241241
proxy_conn_rec *conn; /* Single connection for prefork mpm's */
242242
};
243243

244-
/* woker status flags */
244+
/* worker status flags */
245245
#define PROXY_WORKER_INITIALIZED 0x0001
246246
#define PROXY_WORKER_IGNORE_ERRORS 0x0002
247247
#define PROXY_WORKER_IN_SHUTDOWN 0x0010
248248
#define PROXY_WORKER_DISABLED 0x0020
249249
#define PROXY_WORKER_STOPPED 0x0040
250250
#define PROXY_WORKER_IN_ERROR 0x0080
251+
#define PROXY_WORKER_HOT_STANDBY 0x0100
251252

252-
#define PROXY_WORKER_IS_USABLE(f) (!((f)->s->status & 0x00F0))
253+
#define PROXY_WORKER_NOT_USABLE_BITMAP ( PROXY_WORKER_IN_SHUTDOWN | \
254+
PROXY_WORKER_DISABLED | PROXY_WORKER_STOPPED | PROXY_WORKER_IN_ERROR )
255+
256+
#define PROXY_WORKER_IS_INITIALIZED(f) ( (f)->s->status & \
257+
PROXY_WORKER_INITIALIZED )
258+
259+
#define PROXY_WORKER_IS_STANDBY(f) ( (f)->s->status & \
260+
PROXY_WORKER_HOT_STANDBY )
261+
262+
#define PROXY_WORKER_IS_USABLE(f) ( !((f)->s->status & \
263+
(PROXY_WORKER_NOT_USABLE_BITMAP)) && PROXY_WORKER_IS_INITIALIZED(f) )
253264

254265
/* default worker retry timeout in seconds */
255266
#define PROXY_WORKER_DEFAULT_RETRY 60

‎modules/proxy/mod_proxy_balancer.c‎

Lines changed: 121 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -169,15 +169,63 @@ static char *get_cookie_param(request_rec *r, const char *name)
169169
/* Find the worker that has the 'route' defined
170170
*/
171171
static proxy_worker *find_route_worker(proxy_balancer *balancer,
172-
const char *route)
172+
const char *route, request_rec *r)
173173
{
174174
int i;
175-
proxy_worker *worker = (proxy_worker *)balancer->workers->elts;
176-
for (i = 0; i < balancer->workers->nelts; i++) {
177-
if (*(worker->s->route) && strcmp(worker->s->route, route) == 0) {
178-
return worker;
175+
int checking_standby = 0;
176+
int checked_standby = 0;
177+
178+
proxy_worker *worker;
179+
while (!checked_standby) {
180+
worker = (proxy_worker *)balancer->workers->elts;
181+
for (i = 0; i < balancer->workers->nelts; i++, worker++) {
182+
if ( (checking_standby ? !PROXY_WORKER_IS_STANDBY(worker) : PROXY_WORKER_IS_STANDBY(worker)) )
183+
continue;
184+
if (*(worker->s->route) && strcmp(worker->s->route, route) == 0) {
185+
if (worker && PROXY_WORKER_IS_USABLE(worker)) {
186+
return worker;
187+
} else {
188+
/*
189+
* If the worker is in error state run
190+
* retry on that worker. It will be marked as
191+
* operational if the retry timeout is elapsed.
192+
* The worker might still be unusable, but we try
193+
* anyway.
194+
*/
195+
ap_proxy_retry_worker("BALANCER", worker, r->server);
196+
if (PROXY_WORKER_IS_USABLE(worker)) {
197+
return worker;
198+
} else {
199+
/*
200+
* We have a worker that is unusable.
201+
* It can be in error or disabled, but in case
202+
* it has a redirection set use that redirection worker.
203+
* This enables to safely remove the member from the
204+
* balancer. Of course you will need some kind of
205+
* session replication between those two remote.
206+
*/
207+
if (*worker->s->redirect) {
208+
proxy_worker *rworker = NULL;
209+
rworker = find_route_worker(balancer, worker->s->redirect, r);
210+
/* Check if the redirect worker is usable */
211+
if (rworker && !PROXY_WORKER_IS_USABLE(rworker)) {
212+
/*
213+
* If the worker is in error state run
214+
* retry on that worker. It will be marked as
215+
* operational if the retry timeout is elapsed.
216+
* The worker might still be unusable, but we try
217+
* anyway.
218+
*/
219+
ap_proxy_retry_worker("BALANCER", rworker, r->server);
220+
}
221+
if (rworker && PROXY_WORKER_IS_USABLE(rworker))
222+
return rworker;
223+
}
224+
}
225+
}
226+
}
179227
}
180-
worker++;
228+
checked_standby = checking_standby++;
181229
}
182230
return NULL;
183231
}
@@ -210,42 +258,7 @@ static proxy_worker *find_session_route(proxy_balancer *balancer,
210258
/* We have a route in path or in cookie
211259
* Find the worker that has this route defined.
212260
*/
213-
worker = find_route_worker(balancer, *route);
214-
if (worker && !PROXY_WORKER_IS_USABLE(worker)) {
215-
/*
216-
* If the worker is in error state run
217-
* retry on that worker. It will be marked as
218-
* operational if the retry timeout is elapsed.
219-
* The worker might still be unusable, but we try
220-
* anyway.
221-
*/
222-
ap_proxy_retry_worker("BALANCER", worker, r->server);
223-
if (!PROXY_WORKER_IS_USABLE(worker)) {
224-
/*
225-
* We have a worker that is unusable.
226-
* It can be in error or disabled, but in case
227-
* it has a redirection set use that redirection worker.
228-
* This enables to safely remove the member from the
229-
* balancer. Of course you will need some kind of
230-
* session replication between those two remote.
231-
*/
232-
if (*worker->s->redirect)
233-
worker = find_route_worker(balancer, worker->s->redirect);
234-
/* Check if the redirect worker is usable */
235-
if (worker && !PROXY_WORKER_IS_USABLE(worker)) {
236-
/*
237-
* If the worker is in error state run
238-
* retry on that worker. It will be marked as
239-
* operational if the retry timeout is elapsed.
240-
* The worker might still be unusable, but we try
241-
* anyway.
242-
*/
243-
ap_proxy_retry_worker("BALANCER", worker, r->server);
244-
if (!PROXY_WORKER_IS_USABLE(worker))
245-
worker = NULL;
246-
}
247-
}
248-
}
261+
worker = find_route_worker(balancer, *route, r);
249262
return worker;
250263
}
251264
else
@@ -370,6 +383,9 @@ static int proxy_balancer_pre_request(proxy_worker **worker,
370383
for (i = 0; i < (*balancer)->workers->nelts; i++) {
371384
/* Take into calculation only the workers that are
372385
* not in error state or not disabled.
386+
*
387+
* TODO: Abstract the below, since this is dependent
388+
* on the LB implementation
373389
*/
374390
if (PROXY_WORKER_IS_USABLE(workers)) {
375391
workers->s->lbstatus += workers->s->lbfactor;
@@ -662,12 +678,16 @@ static int balancer_handler(request_rec *r)
662678
ap_rvputs(r, "</td><td>", worker->s->redirect, NULL);
663679
ap_rprintf(r, "</td><td>%d</td><td>", worker->s->lbfactor);
664680
if (worker->s->status & PROXY_WORKER_DISABLED)
665-
ap_rputs("Dis", r);
666-
else if (worker->s->status & PROXY_WORKER_IN_ERROR)
667-
ap_rputs("Err", r);
668-
else if (worker->s->status & PROXY_WORKER_INITIALIZED)
681+
ap_rputs("Dis ", r);
682+
if (worker->s->status & PROXY_WORKER_IN_ERROR)
683+
ap_rputs("Err ", r);
684+
if (worker->s->status & PROXY_WORKER_STOPPED)
685+
ap_rputs("Stop ", r);
686+
if (worker->s->status & PROXY_WORKER_HOT_STANDBY)
687+
ap_rputs("Stby ", r);
688+
if (PROXY_WORKER_IS_USABLE(worker))
669689
ap_rputs("Ok", r);
670-
else
690+
if (!PROXY_WORKER_IS_INITIALIZED(worker))
671691
ap_rputs("-", r);
672692
ap_rputs("</td></tr>\n", r);
673693

@@ -817,34 +837,40 @@ static proxy_worker *find_best_byrequests(proxy_balancer *balancer,
817837
{
818838
int i;
819839
int total_factor = 0;
820-
proxy_worker *worker = (proxy_worker *)balancer->workers->elts;
840+
proxy_worker *worker;
821841
proxy_worker *mycandidate = NULL;
822-
823-
842+
int checking_standby = 0;
843+
int checked_standby = 0;
844+
824845
ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
825846
"proxy: Entering byrequests for BALANCER (%s)",
826847
balancer->name);
827848

828849
/* First try to see if we have available candidate */
829-
for (i = 0; i < balancer->workers->nelts; i++) {
830-
/* If the worker is in error state run
831-
* retry on that worker. It will be marked as
832-
* operational if the retry timeout is elapsed.
833-
* The worker might still be unusable, but we try
834-
* anyway.
835-
*/
836-
if (!PROXY_WORKER_IS_USABLE(worker))
837-
ap_proxy_retry_worker("BALANCER", worker, r->server);
838-
/* Take into calculation only the workers that are
839-
* not in error state or not disabled.
840-
*/
841-
if (PROXY_WORKER_IS_USABLE(worker)) {
842-
worker->s->lbstatus += worker->s->lbfactor;
843-
total_factor += worker->s->lbfactor;
844-
if (!mycandidate || worker->s->lbstatus > mycandidate->s->lbstatus)
845-
mycandidate = worker;
850+
while (!mycandidate && !checked_standby) {
851+
worker = (proxy_worker *)balancer->workers->elts;
852+
for (i = 0; i < balancer->workers->nelts; i++, worker++) {
853+
if ( (checking_standby ? !PROXY_WORKER_IS_STANDBY(worker) : PROXY_WORKER_IS_STANDBY(worker)) )
854+
continue;
855+
/* If the worker is in error state run
856+
* retry on that worker. It will be marked as
857+
* operational if the retry timeout is elapsed.
858+
* The worker might still be unusable, but we try
859+
* anyway.
860+
*/
861+
if (!PROXY_WORKER_IS_USABLE(worker))
862+
ap_proxy_retry_worker("BALANCER", worker, r->server);
863+
/* Take into calculation only the workers that are
864+
* not in error state or not disabled.
865+
*/
866+
if (PROXY_WORKER_IS_USABLE(worker)) {
867+
worker->s->lbstatus += worker->s->lbfactor;
868+
total_factor += worker->s->lbfactor;
869+
if (!mycandidate || worker->s->lbstatus > mycandidate->s->lbstatus)
870+
mycandidate = worker;
871+
}
846872
}
847-
worker++;
873+
checked_standby = checking_standby++;
848874
}
849875

850876
if (mycandidate) {
@@ -878,35 +904,42 @@ static proxy_worker *find_best_bytraffic(proxy_balancer *balancer,
878904
int i;
879905
apr_off_t mytraffic = 0;
880906
apr_off_t curmin = 0;
881-
proxy_worker *worker = (proxy_worker *)balancer->workers->elts;
907+
proxy_worker *worker;
908+
int checking_standby = 0;
909+
int checked_standby = 0;
882910
proxy_worker *mycandidate = NULL;
883911

884912
ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
885913
"proxy: Entering bytraffic for BALANCER (%s)",
886914
balancer->name);
887915

888916
/* First try to see if we have available candidate */
889-
for (i = 0; i < balancer->workers->nelts; i++) {
890-
/* If the worker is in error state run
891-
* retry on that worker. It will be marked as
892-
* operational if the retry timeout is elapsed.
893-
* The worker might still be unusable, but we try
894-
* anyway.
895-
*/
896-
if (!PROXY_WORKER_IS_USABLE(worker))
897-
ap_proxy_retry_worker("BALANCER", worker, r->server);
898-
/* Take into calculation only the workers that are
899-
* not in error state or not disabled.
900-
*/
901-
if (PROXY_WORKER_IS_USABLE(worker)) {
902-
mytraffic = (worker->s->transferred/worker->s->lbfactor) +
903-
(worker->s->read/worker->s->lbfactor);
904-
if (!mycandidate || mytraffic < curmin) {
905-
mycandidate = worker;
906-
curmin = mytraffic;
917+
while (!mycandidate && !checked_standby) {
918+
worker = (proxy_worker *)balancer->workers->elts;
919+
for (i = 0; i < balancer->workers->nelts; i++, worker++) {
920+
if ( (checking_standby ? !PROXY_WORKER_IS_STANDBY(worker) : PROXY_WORKER_IS_STANDBY(worker)) )
921+
continue;
922+
/* If the worker is in error state run
923+
* retry on that worker. It will be marked as
924+
* operational if the retry timeout is elapsed.
925+
* The worker might still be unusable, but we try
926+
* anyway.
927+
*/
928+
if (!PROXY_WORKER_IS_USABLE(worker))
929+
ap_proxy_retry_worker("BALANCER", worker, r->server);
930+
/* Take into calculation only the workers that are
931+
* not in error state or not disabled.
932+
*/
933+
if (PROXY_WORKER_IS_USABLE(worker)) {
934+
mytraffic = (worker->s->transferred/worker->s->lbfactor) +
935+
(worker->s->read/worker->s->lbfactor);
936+
if (!mycandidate || mytraffic < curmin) {
937+
mycandidate = worker;
938+
curmin = mytraffic;
939+
}
907940
}
908941
}
909-
worker++;
942+
checked_standby = checking_standby++;
910943
}
911944

912945
if (mycandidate) {

‎modules/proxy/proxy_util.c‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1605,7 +1605,7 @@ PROXY_DECLARE(void) ap_proxy_initialize_worker_share(proxy_server_conf *conf,
16051605
void *score = NULL;
16061606
#endif
16071607

1608-
if (worker->s && (worker->s->status & PROXY_WORKER_INITIALIZED)) {
1608+
if (worker->s && PROXY_WORKER_IS_INITIALIZED(worker)) {
16091609
/* The worker share is already initialized */
16101610
ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, s,
16111611
"proxy: worker %s already initialized",
@@ -1639,7 +1639,7 @@ PROXY_DECLARE(void) ap_proxy_initialize_worker_share(proxy_server_conf *conf,
16391639
* recheck to see if we've already been here. Possible
16401640
* if proxy is using scoreboard to hold shared stats
16411641
*/
1642-
if (worker->s->status & PROXY_WORKER_INITIALIZED) {
1642+
if (PROXY_WORKER_IS_INITIALIZED(worker)) {
16431643
/* The worker share is already initialized */
16441644
ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, s,
16451645
"proxy: worker %s already initialized",
@@ -1667,7 +1667,7 @@ PROXY_DECLARE(apr_status_t) ap_proxy_initialize_worker(proxy_worker *worker, ser
16671667
int mpm_threads;
16681668
#endif
16691669

1670-
if (worker->status & PROXY_WORKER_INITIALIZED) {
1670+
if (PROXY_WORKER_IS_INITIALIZED(worker)) {
16711671
/* The worker is already initialized */
16721672
return APR_SUCCESS;
16731673
}

0 commit comments

Comments
 (0)