|
|
@@ -20,6 +20,7 @@ var request = require('request'); |
|
|
|
var started = false; |
|
|
|
var unresolved = {}; |
|
|
|
|
|
|
|
var lastUri = '(unstarted)'; |
|
|
|
var passes = 0; |
|
|
|
var kbytes = 0; |
|
|
|
var fails = []; |
|
|
@@ -29,10 +30,10 @@ var stats = []; |
|
|
|
|
|
|
|
// NB times in milliseconds |
|
|
|
var options = { |
|
|
|
loopSleep: 1000, |
|
|
|
loopSleep: 15000, |
|
|
|
verbose: false, |
|
|
|
// Set this to 0 if you're an asshole. |
|
|
|
fetchDelay: 200 |
|
|
|
fetchDelay: 250 |
|
|
|
}; |
|
|
|
|
|
|
|
var log = (options.verbose ? console.log : function () {}); |
|
|
@@ -68,6 +69,7 @@ function fetch(uri, action, extra) { |
|
|
|
// Wait for options.fetchDelay milliseconds, then fire off the |
|
|
|
// fetch. |
|
|
|
Q.delay(options.fetchDelay).then(function() { |
|
|
|
lastUri = uri; |
|
|
|
request( |
|
|
|
{ |
|
|
|
method: 'GET', |
|
|
@@ -108,7 +110,7 @@ function fetch(uri, action, extra) { |
|
|
|
// Handle a failed fetch (request gave back an error, or |
|
|
|
// response code isn't 2xx). |
|
|
|
console.error('failed ' + uri |
|
|
|
+ ': status ' + response.statusCode |
|
|
|
+ ': status ' + (response ? response.statusCode : 'NONE') |
|
|
|
+ ', err ' + error); |
|
|
|
// Stuff the fetch information into the fails list. |
|
|
|
fails.push({ |
|
|
@@ -140,8 +142,9 @@ function fetch(uri, action, extra) { |
|
|
|
*/ |
|
|
|
function watch() { |
|
|
|
var defer = Q.defer(); |
|
|
|
var qlen = Object.keys(unresolved).length; |
|
|
|
|
|
|
|
if (!started || Object.keys(unresolved).length > 0) { |
|
|
|
if (!started || qlen > 0) { |
|
|
|
// If the fetcher hasn't started yet (no fetches queued), or if |
|
|
|
// any fetches are currently in flight, then just report stats, |
|
|
|
// wait, and go around the loop again. |
|
|
@@ -150,8 +153,8 @@ function watch() { |
|
|
|
}).join('') |
|
|
|
+ passes + ' gets, ' |
|
|
|
+ fails.length + ' fails, ' |
|
|
|
+ Math.floor(kbytes/1024) + ' M total' |
|
|
|
); |
|
|
|
+ Math.floor(kbytes/1024) + 'M, ' |
|
|
|
+ 'fetchq ' + qlen); |
|
|
|
|
|
|
|
if (fails.length > 0) { |
|
|
|
console.log('pushing a retry'); |