又是我,新的一天新的爬虫问题。
运行结果,程序能跑起来但是下载率实在没法看,是因为什么问题导致这样,有解决方法吗
贴一下拙劣的代码。
var cheerio = require("cheerio");
http = require("http"),
https = require("https"),
request = require("request");
fs =require("fs"),
path =require("path"),
util = require('util');
var url = 'http://www.ivsky.com/tupian/';
dir = 'D:/download/',
dir = mkdirSync(dir);
function main(){
for(i=0;i<5;i++){
if(i!=0){
var index = 'index_' + i + '.html';
}else{
var index = '';
}
_url = url + index;
//console.log(_url);
downHtml(_url);
}
}
// 获取html
function downHtml(url){
http.get(url, function(res){
var data="";
res.setEncoding("utf-8");
res.on('data', function(chunk){
data += chunk;
});
res.on("end", function(){
var $ = cheerio.load(data);
var allhref=[];
$('.il_img a').each(function(i, e){
allhref[i] = 'http://www.ivsky.com' + $(e).attr('href');
for(i=0;i<allhref.length;i++){
str = encodeURI(allhref[i]);
download(str);
}
})
});
}).on("error", function(){
return null;
});
}
//遍历图片
function download(url){
http.get(url, function(res){
var data="";
res.setEncoding("utf-8");
res.on('data', function(chunk){
data += chunk;
});
res.on("end", function(){
var $ = cheerio.load(data);
$('img').each(function(i, e){
downImg($(e).attr('src'), dir);
})
});
}).on("error", function(){
return null;
});
}
//包装了模块就函数进行复写
/*server.download(url, function(data){
if(data){
//console.log(data);
var $= cheerio.load(data);
$('img').each(function(i, e){
//console.log($(e).attr('src'));
downImg($(e).attr("src"), dir)
})
}
});*/
//创建目录
function mkdirSync(dir){
fs.existsSync(dir) || fs.mkdirSync(dir);
return dir;
}
//下载图片
function downImg(url, dir){
var startTime = new Date().getTime();
url = encodeURI(url);
var fileName = url.split('/').pop();
//var img_name = $('il_img a').attr('title').trim() + '.jpg';
request(url).on('response', function(){
var endTime = new Date().getTime();
console.log('downloading..%s..%ss', url, (endTime-startTime)/1000);
})
.pipe(fs.createWriteStream(dir + fileName)
)
.on('error', function(){console.log('failed to download');});
}
main();//执行主函数