我实现的Node.js中类似Sleep的功能，请大家点评！
最近在抓取一个网站的数据，它有反爬虫策略，不让我持续访问超过100个网络链接，如果访问一段时间暂停一会的话就可以绕过去了，所以就有了下面这段代码，请大家看看，有没有更好的实现方法，首先说明一下，这个小程序运行在命令行下就可以了，我只要抓取数据就可以了。 实现的思路： 主要是通过setTimeout来实现暂停功能，将要抓取的连接保存在一个数组中，将数组进行分组，每抓取完3个连接就暂停5秒钟，并且结果统一保存在一个外部变量results中。
//测试Nodejs怎样实现Sleep的功能
var async = require('async');
//通过setTimeout函数实现Sleep的功能
var sleep = function(array, callback) {
'use strict';
//初始化变量
var results = []; //存放最终结果
var tmpArray = array.slice(0); //拷贝一份数据，防止改变变量外部的值
var isEnd = false;
//自调用函数
var run = function() {
var tmp = null; //存放截取的数组
if (tmpArray.length > 3) {
tmp = tmpArray.splice(0, 3);
} else {
tmp = tmpArray;
isEnd = true;
}
async.mapLimit(tmp, 2, function(item, itemCallback) {
console.log('Enter: ' + item.name);
setTimeout(function() {
console.log('Handle: ' + item.name);
itemCallback(false, item.name + '!!!');
}, item.delay);
}, function(err, data) {
console.log(data);
results = results.concat(data);
if (isEnd) {
callback(false, results);
} else {
console.log('Sleep 5s');
setTimeout(run, 5000);
}
});
};
run();
};
//测试数据
var arr = [
{name:'Jack', delay:2000},
{name:'Mike', delay: 1000},
{name:'Freewind', delay:3000},
{name:'Test', delay: 2500},
{name:'Woody', delay: 3500},
{name:'Water', delay: 2600},
{name:'Sugar', delay: 1500}
];
sleep(arr, function(error, data) {
'use strict';
console.log(data);
});
9 回复
let arr = [
{name:'Jack'},
{name:'Mike'},
{name:'Freewind'},
{name:'Test'},
{name:'Woody'},
{name:'Water'},
{name:'Sugar'}
]
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms))
}
function download(name) {
return new Promise(resolve => setTimeout(
() => resolve('result for ' + name),
Math.floor(Math.random() * 1000)
))
}
(async function main() {
const concurrency = 3;
for (let i = 0; i < arr.length; i+=concurrency) {
console.log(i, '----')
let task = [];
for (let j = 0; j < concurrency && i +j < arr.length; j++) {
task.push(download(arr[i+j].name));
}
var result = await Promise.all(task)
console.log(result)
await sleep(5000)
}
})()
var arr = [
{name:'Jack', delay:2000},
{name:'Mike', delay: 1000},
{name:'Freewind', delay:3000},
{name:'Test', delay: 2500},
{name:'Woody', delay: 3500},
{name:'Water', delay: 2600},
{name:'Sugar', delay: 1500}
];
function sleep(array, callback) {
function next() {
var data = array.shift();
if (data) {
setTimeout(next, data.delay);
callback(null, data.name);
}
}
next();
}
sleep(arr, function (error, data) {
console.log(data);
});
