通过 mongoose save 持久化,发觉数据保存很慢
代码如下
发现 save() 的回调很久才会被调用,命令行 mongo,db.find() 发觉数据增长超级慢,调用了 200+ 次,才有 10,20 条数据被保存下来
发现了 在 stream.pipe() 这个方法的回调里,如果有其他的回调,处理都很慢,有什么方法解决呢?
let save = function(array) {
return new Promise(function(resolve, reject) {
console.log('will save books')
Book.save(array, function (err, list) {
if (err) {
console.log('Fail')
resolve()
} else {
console.log('Save 100')
reslove()
}
})
})
}
co(function*(){
yield conn()
yield sentStream()
let stream = yield isbnStream()
let count = 0
let array = []
stream.pipe(es.mapSync(function (data) {
delete data.score
delete data._id
count += 1
let isbn = data['isbn']
let title = data['title']
let comments = data['comments']
let reviews = data['reviews']
if (comments.length < 1 && reviews.length < 1)
return
let objectList = comments.concat(reviews)
console.log('Count: ' + count + ' Title: ' + title)
let finalScore = scoreSentences(objectList)
let book = new Book(data)
book.score = finalScore
array.push(book)
if (array.length % 1 === 0) {
save(array)
array = []
}
}))
})