kettle/lib/dataSource-url.js

/*!
Kettle File DataSource

Copyright 2012-2013 OCAD University
Copyright 2016 OCAD University

Licensed under the New BSD license. You may not use this file except in
compliance with this License.

You may obtain a copy of the License at
https://github.com/fluid-project/kettle/blob/master/LICENSE.txt
*/

"use strict";

var fluid = fluid || require("infusion"),
    kettle = fluid.registerNamespace("kettle"),
    http = http || require("http"),
    https = https || require("https"),
    urlModule = urlModule || require("url");

fluid.registerNamespace("kettle.dataSource");

/**** URL DATASOURCE SUPPORT ****/

fluid.defaults("kettle.dataSource.URL", {
    gradeNames: ["kettle.dataSource"],
    readOnlyGrade: "kettle.dataSource.URL",
    invokers: {
        resolveUrl: "kettle.dataSource.URL.resolveUrl", // url, termMap, directModel, noencode
        getImpl: {
            funcName: "kettle.dataSource.URL.handle",
            args: ["{that}", "{arguments}.0", "{arguments}.1"] // options, directModel
        }
    },
    censorRequestOptionsLog: {
        auth: true,
        "headers.Authorization": true
    },
    components: {
        cookieJar: "{cookieJar}"
    },
    termMap: {}
});

fluid.defaults("kettle.dataSource.URL.writable", {
    gradeNames: ["kettle.dataSource.writable"],
    invokers: {
        setImpl: {
            funcName: "kettle.dataSource.URL.handle",
            args: ["{that}", "{arguments}.0", "{arguments}.1", "{arguments}.2"] // options, directModel, model
        }
    }
});

/**
 * Resolves (expands) a url or path with respect to the `directModel` supplied to a dataSource's API (get or set). There are three rounds of expansion - firstly, the string
 * entries as the values in "termMap" will be looked up as paths within `directModel`. The resulting values will then be URI Encoded, unless their value
 * the termMap is prefixed with `noencode:`. Secondly,
 * this resolved termMap will be used for a round of the standard fluid.stringTemplate algorithm applied to the supplied URL. Finally, any argument `expand` will be
 * used to expand the entire URL.
 * @param {String} url - A url or path to expand
 * @param {Object.<String, String>} termMap - A map of terms to be used for string interpolation. Any values which begin with the prefix `noencode:` will have this prefix stripped off, and
 * URI encoding will not be applied to the substituted value. After the value is normalised in this way, the remaining value may be used for indirection in the directModel if it
 * begins with the prefix "%", or else directly for interpolation.
 * @param {Object} directModel - a set of data used to expand the url template
 * @param {Boolean} noencode - `true` if URLencoding of each interpolated value in the URL should be defeated by default
 * @return {String} The resolved/expanded url
 */
kettle.dataSource.URL.resolveUrl = function (url, termMap, directModel, noencode) {
    var map = fluid.transform(termMap, function resolve(entry) {
        entry = String(entry);
        var encode = !noencode;
        if (entry.indexOf("noencode:") === 0) {
            encode = false;
            entry = entry.substring("noencode:".length);
        }
        var value = entry.charAt(0) === "%" ? fluid.get(directModel, entry.substring(1)) : entry;
        if (encode) {
            value = encodeURIComponent(value);
        }
        return value;
    });
    var replaced = fluid.stringTemplate(url, map);
    return replaced;
};

kettle.dataSource.URL.requestOptions = ["url", "protocol", "host", "hostname", "family", "port", "localAddress", "socketPath", "method", "path", "headers", "auth", "agent", "termMap"];

// TODO: Deal with the anomalous status of "charEncoding" - in theory it could be set per-request but currently can't be. Currently all
// "requestOptions" have a common fate in that they end up as the arguments to http.request. We need to split these into two levels,
// httpRequestOptions and the outer level with everything else. We also need to do something similar for kettle.dataSource.file

/** Assemble the `requestOptions` structure that will be sent to `http.request` by `kettle.dataSource.URL` by fusing together values from the user, the component
 * with filtration by a list of permitted options, e.g. those listed in `kettle.dataSource.URL.requestOptions`. A poorly factored method that needs to be
 * reformed as a proper merge pipeline.
 */

kettle.dataSource.URL.prepareRequestOptions = function (componentOptions, cookieJar, userOptions, permittedOptions, directModel, userStaticOptions, resolveUrl) {
    var staticOptions = fluid.filterKeys(componentOptions, permittedOptions);
    var requestOptions = fluid.extend(true, {headers: {}}, userStaticOptions, staticOptions, userOptions);
    // GPII-2147: replace "localhost" with "127.0.0.1" to allow running without a network connection in windows
    if (requestOptions.hostname === "localhost") {
        requestOptions.hostname = "127.0.0.1";
    }
    if (requestOptions.host === "localhost") {
        requestOptions.host = "127.0.0.1";
    }
    var termMap = fluid.transform(requestOptions.termMap, encodeURIComponent);

    requestOptions.path = (resolveUrl || kettle.dataSource.URL.resolveUrl)(requestOptions.path, requestOptions.termMap, directModel);

    fluid.stringTemplate(requestOptions.path, termMap);
    if (cookieJar && cookieJar.cookie && componentOptions.storeCookies) {
        requestOptions.headers.Cookie = cookieJar.cookie;
    }
    return requestOptions;
};

/** Given an HTTP status code as returned by node's `http.IncomingMessage` class (or otherwise), determine whether it corresponds to
 * an error status. This performs a simple-minded check to see if it a number outside the range [200, 300).
 * @param {Number} statusCode The HTTP status code to be tested
 * @return {Boolean} `true` if the status code represents an error status
 */

kettle.dataSource.URL.isErrorStatus = function (statusCode) {
    return statusCode < 200 || statusCode >= 300;
};

/**
 * Handles calls to a URL data source's get and set.
 * @param {kettle.dataSource.urlResolver} that - A URLResolver that will convert the contents of the
 * <code>directModel</code> supplied as the 3rd argument into a concrete URL used for this
 * HTTP request.
 * @param {Object} userOptions - An options block that encodes:
 *     @param {String}  userOptions.operation - "set"/"get"
 *     @param {Boolean} userOptions.notFoundIsEmpty - <code>true</code> if a missing file on read should count as a successful empty payload rather than a rejection
 *     writeMethod {String}: "PUT"/ "POST" (optional - if not provided will be defaulted by the concrete dataSource implementation)
 * @param {Object} directModel - a model holding the coordinates of the data to be read or written
 * @param {Object} [model] - [optional] the payload to be written by this write operation
 * @return {Promise} a promise for the successful or failed datasource operation
 */
kettle.dataSource.URL.handle = function (that, userOptions, directModel, model) {
    var permittedOptions = kettle.dataSource.URL.requestOptions;
    var url = that.resolveUrl(that.options.url, that.options.termMap, directModel);
    var parsed = fluid.filterKeys(urlModule.parse(url, true), permittedOptions);
    var requestOptions = kettle.dataSource.URL.prepareRequestOptions(that.options, that.cookieJar, userOptions, permittedOptions, directModel, parsed);

    return kettle.dataSource.URL.handle.http(that, requestOptions, model);
};

// Attempt to parse the error response as JSON, but if failed, just stick it into "message" quietly
kettle.dataSource.URL.relayError = function (response, received, whileMsg) {
    var rejectPayload;
    try {
        rejectPayload = JSON.parse(received);
    } catch (e) {
        var message = typeof(received) === "string" && received.indexOf("<html") !== -1 ?
            kettle.extractHtmlError(received) : received;
        rejectPayload = {message: message};
    }
    rejectPayload.message = rejectPayload.message || rejectPayload.error;
    delete rejectPayload.error;
    rejectPayload.isError = true;
    rejectPayload.statusCode = response.statusCode;
    return kettle.upgradeError(rejectPayload, whileMsg);
};

/** Compute the callback to be supplied to `http.request` for this dataSource operation
 * @param requestOptions {Object} The fully merged options which were sent to `http.request`
 * @param promise {Promise} The outgoing promise to be returned to the user
 * @param whileMsg {String} A readable summary of the current operation (including HTTP method and URL) to be suffixed to any rejection message
 * @return {Function} The callback to be supplied to `http.request`
 */

kettle.dataSource.URL.httpCallback = function (requestOptions, promise, whileMsg) {
    return function (res) {
        var received = "";
        res.setEncoding(requestOptions.charEncoding);
        res.on("data", function onData(chunk) {
            received += chunk;
        });
        res.on("end", kettle.wrapCallback(
            function () {
                if (kettle.dataSource.URL.isErrorStatus(res.statusCode)) {
                    var relayed = kettle.dataSource.URL.relayError(res, received, whileMsg);
                    if (requestOptions.notFoundIsEmpty && relayed.statusCode === 404) {
                        promise.resolve(undefined);
                    } else {
                        promise.reject(relayed);
                    }
                } else {
                    promise.resolve(received);
                }
            })
        );
    };
};

/** Compute the listener to be attached to the `http.request` `error` event for this dataSource operation
 * @param promise {Promise} The outgoing promise to be returned to the user
 * @param whileMsg {String} A readable summary of the current operation (including HTTP method and URL) to be suffixed to any rejection message
 */

kettle.dataSource.URL.errorCallback = function (promise, whileMsg) {
    return function (error) {
        error.isError = true;
        promise.reject(kettle.upgradeError(error, whileMsg));
    };
};

/** Prepare a URL for logging by censoring sensitive parts of the URL (satisfy KETTLE-73, GPII-3309)
 * @param requestOptions {Object} A hash of request options holding a set of parsed URL fields as returned from
 * https://nodejs.org/api/url.html#url_url_parse_urlstring_parsequerystring_slashesdenotehost , as well as some others
 * including the overall `url`.
 * @param toCensor {Object} A hash of member paths within `requestOptions` to `true`, holding those which should be censored
 * @return A requestOptions object with the sensitive members removed where they appear at top level as well as where they might occur encoded
 * within the `url` member
 */

kettle.dataSource.URL.censorRequestOptions = function (requestOptions, toCensor) {
    var togo = fluid.copy(requestOptions);
    fluid.each(toCensor, function (troo, key) {
        var original = fluid.get(togo, key);
        if (original) {
            fluid.set(togo, key, "(SENSITIVE)");
        }
    });
    // Compensate for our use of legacy member "path" throughout
    var parsedPath = urlModule.parse(togo.path);
    togo.pathname = parsedPath.pathname;
    togo.search = parsedPath.search;
    togo.url = urlModule.format(togo);
    return togo;
};

/** Central strategy point for all HTTP-backed DataSource operations (both read and write).
 * Accumulates options to be sent to the underlying node.js `http.request` primitives, collects and interprets the
 * results back into promise resolutions.
 * @param that {Component} The DataSource itself
 * @param baseOptions {Object} A partially merged form of the options sent to the top-level `dataSource.get` method together with relevant
 * static options configured on the component. Information in the `directModel` argument has already been encoded into the url member.
 * @param data {String} The `model` argument sent to top-level `dataSource.get/set` after it has been pushed through the transform chain
 */

kettle.dataSource.URL.handle.http = function (that, baseOptions, data) {
    var promise = fluid.promise();
    var defaultOptions = {
        port: 80,
        method: "GET"
    };

    var dataBuffer;

    if (baseOptions.operation === "set") {
        dataBuffer = new Buffer(data);
        defaultOptions.headers = {
            "Content-Type": that.encoding.options.contentType,
            "Content-Length": dataBuffer.length
        };
        defaultOptions.method = baseOptions.writeMethod;
    }
    var requestOptions = fluid.extend(true, defaultOptions, baseOptions);
    var loggingOptions = kettle.dataSource.URL.censorRequestOptions(requestOptions, that.options.censorRequestOptionsLog);
    var whileMsg = " while executing HTTP " + requestOptions.method + " on url " + loggingOptions.url;
    fluid.log("DataSource Issuing " + (requestOptions.protocol.toUpperCase()).slice(0, -1) + " request with options ",
        loggingOptions);
    promise.accumulateRejectionReason = function (originError) {
        return kettle.upgradeError(originError, whileMsg);
    };
    var callback = kettle.wrapCallback(kettle.dataSource.URL.httpCallback(requestOptions, promise, whileMsg));
    var req;
    if (requestOptions.protocol === "http:") {
        req = http.request(requestOptions, callback);
    } else if (requestOptions.protocol === "https:") {
        req = https.request(requestOptions, callback);
    } else {
        fluid.fail("kettle.dataSource.URL cannot handle unknown protocol "  + requestOptions.protocol);
    }
    req.on("error", kettle.wrapCallback(kettle.dataSource.URL.errorCallback(promise, whileMsg)));
    req.end(dataBuffer ? dataBuffer : data);
    return promise;
};