使用superagent 爬网站的时候使用async 对并发个数进行控制,并发个数确保了,但是数据的结果重复了:爬取的数组长度是10,最后结果输出了10次,求教该如何控制输出结果不重复(或者说如何判断全部url爬完)? 代码如下:
var eventproxy = require('eventproxy');
var superagent = require('superagent');
var cheerio = require('cheerio');
var url = require('url');
var async = require('async');
var fs = require('fs');
var itjuziUrl = 'http://www.itjuzi.com/company/';
var companyUrls = [];
for (i = 1; i <= 10; i++) {
var href = url.resolve(itjuziUrl, i.toString()); //
companyUrls.push(href);
}
var companyJson = [];
var concurrencyCount = 0;
var fetchUrl= function(url ,callback){
companyurl = url;
concurrencyCount ++ ;
superagent.get(companyurl)
.end(function (err, res) {
if(err){
callback(err, 'fetch url:' + companyurl + ' err: ');
return console.log('fetch url:' + companyurl + ' err: ', err);
}
console.log('现在并发数目是******: ',concurrencyCount,' **** fetch ' + companyurl + ' successful');
//ep.emit('topic_html', [companyurl, res.text]);
var $ = cheerio.load(res.text);
concurrencyCount --;
companyJson.push({
productName: $('div>span.title').text().replace(/[\n\t\r\s]/g,""),
companyName: $('div.des-more>div> span').eq(0).text().replace(/[\n\t\r\s]/g,""),
address: $('span.loca.c-gray-aset').text().replace(/[\n\t\r\s]/g,""),
stage: $('td.mobile-none').text().replace(/[\n\t\r\s]/g,""),
mainpage: $('a.weblink').attr('href'),
hirepage: ''
});
callback(null, companyJson);
});
}
async.mapLimit(companyUrls , 5, function(url,callback){
fetchUrl(url ,callback)
},function(err, result ){
//console.log('now err , result is ',result);
if (!err) {
console.log('result.length ',result.length );
if (companyUrls.length = result.length) {
arr = result;
var file = fs.createWriteStream('result.json');
file.on('error', function (err) {
console.log('write file error ', err);
});
arr.forEach(function (v) {
//console.log('v is ', JSON.stringify(v));
file.write(JSON.stringify(v) + '\n');
});
file.end();
}
}
})
\n```