Skip to content

Commit 2ce6853

Browse files
PlaneshifterPlaneshifter
committed
Merge branch 'develop' of https://github.com/stdlib-js/stdlib into develop
2 parents b19c395 + c2d4b39 commit 2ce6853

2 files changed

Lines changed: 71 additions & 0 deletions

File tree

‎lib/node_modules/@stdlib/_tools/github/dependants/lib/resolve.js‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ var objectValues = require( '@stdlib/utils/values' );
2626
var substringAfter = require( '@stdlib/string/substring-after' );
2727
var replace = require( '@stdlib/string/base/replace' );
2828
var reWhitespace = require( '@stdlib/regexp/whitespace' );
29+
var exp2 = require( '@stdlib/math/base/special/exp2' );
2930
var getOptions = require( './options.js' );
3031
var request = require( './request.js' );
3132
var urlpath = require( './path.js' );
@@ -34,10 +35,15 @@ var urlpath = require( './path.js' );
3435
// VARIABLES //
3536

3637
var debug = logger( 'github-dependants:resolve' );
38+
3739
var RE_WHITESPACE = reWhitespace({
3840
'flags': 'g'
3941
});
4042

43+
// Specify the maximum number of retries in case we either encounter rate limiting (Too Many Requests) or service unavailable HTTP errors:
44+
var MAX_RETRIES = 10;
45+
var DELAY = 1000;
46+
4147

4248
// MAIN //
4349

@@ -51,12 +57,14 @@ var RE_WHITESPACE = reWhitespace({
5157
*/
5258
function resolve( opts, clbk ) {
5359
var options;
60+
var retries;
5461
var data;
5562

5663
options = getOptions( opts ); // request options
5764
options.path = urlpath( opts.pathname, opts.type );
5865

5966
data = Object.create( null ); // no prototype
67+
retries = 0;
6068

6169
debug( 'Resolving dependants.' );
6270
return next();
@@ -86,8 +94,25 @@ function resolve( opts, clbk ) {
8694
var $;
8795
var i;
8896
if ( error ) {
97+
if (
98+
error.status === 429 || // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429
99+
error.status === 503 || // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503
100+
error.message.indexOf( 'Too Many Requests' ) >= 0 ||
101+
error.message.indexOf( 'Service Unavailable' ) >= 0
102+
) {
103+
retries += 1;
104+
if ( retries > MAX_RETRIES ) {
105+
debug( 'Maximum number of retries exceeded. Status: %d. Error: %s', error.status, error.message );
106+
return done( error );
107+
}
108+
debug( 'Encountered an error. Status: %d. Error: %s', error.status, error.message );
109+
return setTimeout( retry, exp2( retries ) * DELAY );
110+
}
89111
return done( error );
90112
}
113+
// Reset the retry counter:
114+
retries = 0;
115+
91116
$ = cheerio.load( body );
92117
selection = $( 'div.Box-row' );
93118

@@ -151,6 +176,16 @@ function resolve( opts, clbk ) {
151176
}
152177
}
153178

179+
/**
180+
* Retries a request.
181+
*
182+
* @private
183+
*/
184+
function retry() {
185+
debug( 'Retrying request (%d of %d).', retries, MAX_RETRIES );
186+
next();
187+
}
188+
154189
/**
155190
* Callback invoked upon resolving resources.
156191
*

‎lib/node_modules/@stdlib/_tools/npm/pkg-dependants/lib/resolve.js‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
var logger = require( 'debug' );
2424
var cheerio = require( 'cheerio' );
2525
var objectKeys = require( '@stdlib/utils/keys' );
26+
var exp2 = require( '@stdlib/math/base/special/exp2' );
2627
var getOptions = require( './options.js' );
2728
var request = require( './request.js' );
2829
var urlpath = require( './path.js' );
@@ -31,8 +32,14 @@ var urlpath = require( './path.js' );
3132
// VARIABLES //
3233

3334
var debug = logger( 'npm:pkg-dependants:resolve' );
35+
36+
// Specify the "stride" (i.e., maximum number of displayed package dependants on the HTML page; i.e., results per page):
3437
var STRIDE = 36;
3538

39+
// Specify the maximum number of retries in case we either encounter rate limiting (Too Many Requests) or service unavailable HTTP errors:
40+
var MAX_RETRIES = 10;
41+
var DELAY = 1000;
42+
3643

3744
// MAIN //
3845

@@ -46,11 +53,13 @@ var STRIDE = 36;
4653
*/
4754
function resolve( opts, clbk ) {
4855
var options;
56+
var retries;
4957
var offset;
5058
var cache;
5159

5260
options = getOptions( opts ); // request options
5361
offset = 0;
62+
retries = 0;
5463

5564
cache = Object.create( null ); // no prototype
5665

@@ -83,8 +92,25 @@ function resolve( opts, clbk ) {
8392
var selection;
8493
var $;
8594
if ( error ) {
95+
if (
96+
error.status === 429 || // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429
97+
error.status === 503 || // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503
98+
error.message.indexOf( 'Too Many Requests' ) >= 0 ||
99+
error.message.indexOf( 'Service Unavailable' ) >= 0
100+
) {
101+
retries += 1;
102+
if ( retries > MAX_RETRIES ) {
103+
debug( 'Maximum number of retries exceeded. Status: %d. Error: %s', error.status, error.message );
104+
return done( error );
105+
}
106+
debug( 'Encountered an error. Status: %d. Error: %s', error.status, error.message );
107+
return setTimeout( retry, exp2( retries ) * DELAY );
108+
}
86109
return done( error );
87110
}
111+
// Reset the retry counter:
112+
retries = 0;
113+
88114
$ = cheerio.load( body );
89115
selection = $( 'a[href^="/package/"]' );
90116

@@ -112,6 +138,16 @@ function resolve( opts, clbk ) {
112138
}
113139
}
114140

141+
/**
142+
* Retries a request.
143+
*
144+
* @private
145+
*/
146+
function retry() {
147+
debug( 'Retrying request (%d of %d).', retries, MAX_RETRIES );
148+
next( offset );
149+
}
150+
115151
/**
116152
* Callback invoked upon resolving resources.
117153
*

0 commit comments

Comments
 (0)