记一次Node爬取古诗文网 数据#

• 搭建环境

---

1. 使用的是node的koa框架，使用koa的脚手架工具在本地运行起来

const Koa = require("koa");
const app = new Koa();
const static = require("koa-static");
const path = require("path");
const cors = require("koa-cors");
const bodyParser = require('koa-bodyparser');
var history = require('connect-history-api-fallback');

const fs = require('fs');

var request = require('request');

const session = require('koa-session');

app.keys = ['some secret hurr'];

//mongoose数据库

const mongoose = require("mongoose");
mongoose.connect('mongodb://*.*.*.*:27017/koa-mongo');
var db = mongoose.connection;
db.on('error', console.error.bind(console, 'connection error:'));
db.once('open', function() {
  console.log("connnected!!!");
});

// app.use(enforceHttps());
var options = {
    key: fs.readFileSync('guhan.site.key'),
    cert: fs.readFileSync('guhan.site.crt')
};

//静态目录
const staticPath = "./static";


app.use(static(
		path.join(__dirname,staticPath)
	));
app.use(session(CONFIG, app));
app.use(bodyParser());//数据JSON类型

//跨域资源共享
app.use(cors({
    origin:function(ctx){
      // if(ctx.header.origin.indexOf("localhost")>-1){
      //   return ctx.header.origin
      // }else{
      //   return '*'
      // }
      return "*"
    },
    exposeHeaders: ['WWW-Authenticate', 'Server-Authorization', 'Date'],
    maxAge: 100,
    credentials: true,
    allowMethods: ['GET', 'POST', 'OPTIONS'],
    allowHeaders: ['Content-Type', 'Authorization', 'Accept', 'X-Custom-Header', 'anonymous'],
}));

// 递归路由开始
var rootRoute = path.join(__dirname,'/router/');
function utilRouter(rootRoute){
  var file = fs.readdirSync(rootRoute);
  for (var i = 0; i < file.length; i++) {
    var item = file[i];
    var routerPath = path.join(rootRoute,item);
    var stats = fs.statSync(routerPath);
    if(stats.isDirectory()){
      utilRouter(routerPath)
    }else{
       app.use(require(routerPath).routes())
         .use(require(routerPath).allowedMethods())
    }
  }
}

utilRouter(rootRoute);



	
app.listen(80)

• 下载需要的npm包

---

1. superagent和cheerio

    npm install cheerio superagent --save-dev
   

2. superagent是nodejs里一个非常方便的客户端请求代理模块(类似python之中的request模块)当你想处理get,post,put,delete,head请求时,你就应该想起该用它了。详细使用可以查看superagent；

const superagent = require('superagent');
 
// callback
superagent
  .post('/api/pet')
  .send({ name: 'Manny', species: 'cat' }) // sends a JSON post body
  .set('X-API-Key', 'foobar')
  .set('accept', 'json')
  .end((err, res) => {
    // Calling the end function will send the request
  });
 
// promise with then/catch
superagent.post('/api/pet').then(console.log).catch(console.error);
 
// promise with async/await
(async () => {
  try {
    const res = await superagent.post('/api/pet');
    console.log(res);
  } catch (err) {
    console.error(err);
  }
})();

3. cheerio是一个类似Jquery的框架，更多使用查看cheerio

    let $ = cheerio.load('<ul id="fruits">...</ul>');
    
   

移动地址：演示