ezs-istex/lib/scroll-merge.js

'use strict';

Object.defineProperty(exports, "__esModule", {
    value: true
});

var _regenerator = require('babel-runtime/regenerator');

var _regenerator2 = _interopRequireDefault(_regenerator);

var _extends2 = require('babel-runtime/helpers/extends');

var _extends3 = _interopRequireDefault(_extends2);

var _asyncToGenerator2 = require('babel-runtime/helpers/asyncToGenerator');

var _asyncToGenerator3 = _interopRequireDefault(_asyncToGenerator2);

/**
 * Take a string containing a query and outputs records from the ISTEX API.
 *
 * @example
 * from([{ query: 'this is a test' }])
 *   .pipe(ezs('ISTEXScroll', {
 *       maxPage: 2,
 *       size: 1,
 *       sid: 'test',
 *   }))
 * @name ISTEXScroll
 * @param {string} [query=input]          ISTEX query
 * @param {string} [sid="ezs-istex"]    User-agent identifier
 * @param {number} maxPage              Maximum number of pages to get
 * @param {number} [size=2000]          size of each page of results
 * @param {string} [duration="5m"]     maximum duration between two requests
 * @param {Array<string>} [field=["doi"]]   fields to get
 * @returns {Array<Object>}
 */
var ISTEXScrollMerge = function () {
    var _ref = (0, _asyncToGenerator3.default)( /*#__PURE__*/_regenerator2.default.mark(function _callee(data, feed) {
        var query, sid, maxPage, size, scroll, field, fields, output, location, parameters, urlObj, urlStr, response, json, nbPages, prevScrollId, i, _json, nextScrollURI, scrollId, res;

        return _regenerator2.default.wrap(function _callee$(_context) {
            while (1) {
                switch (_context.prev = _context.next) {
                    case 0:
                        if (!this.isLast()) {
                            _context.next = 2;
                            break;
                        }

                        return _context.abrupt('return', feed.close());

                    case 2:
                        query = this.getParam('query') || data.query || '';
                        sid = this.getParam('sid', 'ezs-istex');
                        maxPage = Number(this.getParam('maxPage'));
                        size = Number(this.getParam('size', 2000));
                        scroll = this.getParam('duration', '5m');
                        field = this.getParam('field', ['doi']);
                        fields = Array.isArray(field) ? field : [field];
                        output = 'arkIstex,' + fields.map(function (e) {
                            return (/\w+/.exec(e)[0]
                            );
                        }).join();
                        location = {
                            protocol: 'https:',
                            host: 'api.istex.fr',
                            pathname: '/document/'
                        };
                        parameters = {
                            q: query,
                            scroll: scroll,
                            output: output,
                            size: size,
                            sid: sid
                        };
                        urlObj = (0, _extends3.default)({}, location, {
                            search: _qs2.default.stringify(parameters)
                        });
                        urlStr = _url2.default.format(urlObj);
                        _context.next = 16;
                        return (0, _fetchWithProxy2.default)(urlStr);

                    case 16:
                        response = _context.sent;
                        _context.next = 19;
                        return response.json();

                    case 19:
                        json = _context.sent;

                        if (!(json.total === 0)) {
                            _context.next = 22;
                            break;
                        }

                        return _context.abrupt('return', feed.send(new Error('No result.')));

                    case 22:
                        if (!json._error) {
                            _context.next = 24;
                            break;
                        }

                        return _context.abrupt('return', feed.send(new Error(json._error)));

                    case 24:
                        if (!(json.total === undefined)) {
                            _context.next = 26;
                            break;
                        }

                        return _context.abrupt('return', feed.send(new Error('Unexpected response.')));

                    case 26:
                        nbPages = Math.ceil(json.total / size);

                        if (nbPages > maxPage) {
                            nbPages = maxPage;
                        }
                        if (json.noMoreScrollResults) {
                            nbPages = 1;
                        }
                        prevScrollId = '';
                        i = 0;

                    case 31:
                        if (!(i < nbPages)) {
                            _context.next = 47;
                            break;
                        }

                        _json = json, nextScrollURI = _json.nextScrollURI, scrollId = _json.scrollId;

                        if (process.env.DEBUG) {
                            // eslint-disable-next-line no-console
                            console.assert(prevScrollId === scrollId, 'New ScrollId: ' + scrollId);
                            prevScrollId = scrollId;
                        }
                        json = (0, _extends3.default)({}, json, data);
                        feed.write(json);

                        if (!json.noMoreScrollResults) {
                            _context.next = 38;
                            break;
                        }

                        return _context.abrupt('break', 47);

                    case 38:
                        _context.next = 40;
                        return (0, _fetchWithProxy2.default)(nextScrollURI);

                    case 40:
                        res = _context.sent;
                        _context.next = 43;
                        return res.json();

                    case 43:
                        json = _context.sent;

                    case 44:
                        i += 1;
                        _context.next = 31;
                        break;

                    case 47:

                        feed.end();

                    case 48:
                    case 'end':
                        return _context.stop();
                }
            }
        }, _callee, this);
    }));

    return function ISTEXScrollMerge(_x, _x2) {
        return _ref.apply(this, arguments);
    };
}(); /* eslint-disable no-await-in-loop */


var _url = require('url');

var _url2 = _interopRequireDefault(_url);

var _qs = require('qs');

var _qs2 = _interopRequireDefault(_qs);

var _fetchWithProxy = require('fetch-with-proxy');

var _fetchWithProxy2 = _interopRequireDefault(_fetchWithProxy);

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

exports.default = {
    ISTEXScrollMerge: ISTEXScrollMerge
};