1 | 'use strict'
|
2 | const cheerio = require('cheerio')
|
3 | const got = require('got')
|
4 | const HttpsProxyAgent = require('https-proxy-agent')
|
5 |
|
6 | module.exports = function (url, opts) {
|
7 | if (typeof url !== 'string') {
|
8 | throw new TypeError('Expected a string')
|
9 | }
|
10 |
|
11 | opts = opts || {}
|
12 | const proxy = opts.proxy === undefined ?
|
13 | process.env.http_proxy :
|
14 | opts.proxy
|
15 |
|
16 | const cheerioOpts = Object.assign({
|
17 |
|
18 | decodeEntities: false
|
19 | }, opts.cheerio)
|
20 |
|
21 | const gotOptions = Object.assign({
|
22 | agent: proxy && (new HttpsProxyAgent(proxy))
|
23 | }, opts.got)
|
24 |
|
25 | const ret = got.get(url, gotOptions).then(data => {
|
26 | if (data.statusCode !== 200) {
|
27 | return Promise.reject(new Error(data.statusText))
|
28 | }
|
29 | return data.body
|
30 | })
|
31 | if (opts.htmlOnly) {
|
32 | return ret
|
33 | }
|
34 | return ret.then(data => cheerio.load(data, cheerioOpts))
|
35 | }
|