vega-loader/build/vega-loader.node.js

'use strict';

Object.defineProperty(exports, '__esModule', { value: true });

var vegaUtil = require('vega-util');
var d3Dsv = require('d3-dsv');
var topojsonClient = require('topojson-client');
var vegaFormat = require('vega-format');

// Matches absolute URLs with optional protocol
//   https://...    file://...    //...
const protocol_re = /^([A-Za-z]+:)?\/\//;

// Matches allowed URIs. From https://github.com/cure53/DOMPurify/blob/master/src/regexp.js with added file://
const allowed_re = /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp|file|data):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i; // eslint-disable-line no-useless-escape
const whitespace_re = /[\u0000-\u0020\u00A0\u1680\u180E\u2000-\u2029\u205f\u3000]/g; // eslint-disable-line no-control-regex


// Special treatment in node.js for the file: protocol
const fileProtocol = 'file://';

/**
 * Factory for a loader constructor that provides methods for requesting
 * files from either the network or disk, and for sanitizing request URIs.
 * @param {function} fetch - The Fetch API for HTTP network requests.
 *   If null or undefined, HTTP loading will be disabled.
 * @param {object} fs - The file system interface for file loading.
 *   If null or undefined, local file loading will be disabled.
 * @return {function} A loader constructor with the following signature:
 *   param {object} [options] - Optional default loading options to use.
 *   return {object} - A new loader instance.
 */
function loaderFactory(fetch, fs) {
  return options => ({
    options: options || {},
    sanitize: sanitize,
    load: load,
    fileAccess: !!fs,
    file: fileLoader(fs),
    http: httpLoader(fetch)
  });
}

/**
 * Load an external resource, typically either from the web or from the local
 * filesystem. This function uses {@link sanitize} to first sanitize the uri,
 * then calls either {@link http} (for web requests) or {@link file} (for
 * filesystem loading).
 * @param {string} uri - The resource indicator (e.g., URL or filename).
 * @param {object} [options] - Optional loading options. These options will
 *   override any existing default options.
 * @return {Promise} - A promise that resolves to the loaded content.
 */
async function load(uri, options) {
  const opt = await this.sanitize(uri, options),
        url = opt.href;

  return opt.localFile
    ? this.file(url)
    : this.http(url, options);
}

/**
 * URI sanitizer function.
 * @param {string} uri - The uri (url or filename) to sanity check.
 * @param {object} options - An options hash.
 * @return {Promise} - A promise that resolves to an object containing
 *  sanitized uri data, or rejects it the input uri is deemed invalid.
 *  The properties of the resolved object are assumed to be
 *  valid attributes for an HTML 'a' tag. The sanitized uri *must* be
 *  provided by the 'href' property of the returned object.
 */
async function sanitize(uri, options) {
  options = vegaUtil.extend({}, this.options, options);

  const fileAccess = this.fileAccess,
        result = {href: null};

  let isFile, loadFile, base;

  const isAllowed = allowed_re.test(uri.replace(whitespace_re, ''));

  if (uri == null || typeof uri !== 'string' || !isAllowed) {
    vegaUtil.error('Sanitize failure, invalid URI: ' + vegaUtil.stringValue(uri));
  }

  const hasProtocol = protocol_re.test(uri);

  // if relative url (no protocol/host), prepend baseURL
  if ((base = options.baseURL) && !hasProtocol) {
    // Ensure that there is a slash between the baseURL (e.g. hostname) and url
    if (!uri.startsWith('/') && base[base.length-1] !== '/') {
      uri = '/' + uri;
    }
    uri = base + uri;
  }

  // should we load from file system?
  loadFile = (isFile = uri.startsWith(fileProtocol))
    || options.mode === 'file'
    || options.mode !== 'http' && !hasProtocol && fileAccess;

  if (isFile) {
    // strip file protocol
    uri = uri.slice(fileProtocol.length);
  } else if (uri.startsWith('//')) {
    if (options.defaultProtocol === 'file') {
      // if is file, strip protocol and set loadFile flag
      uri = uri.slice(2);
      loadFile = true;
    } else {
      // if relative protocol (starts with '//'), prepend default protocol
      uri = (options.defaultProtocol || 'http') + ':' + uri;
    }
  }

  // set non-enumerable mode flag to indicate local file load
  Object.defineProperty(result, 'localFile', {value: !!loadFile});

  // set uri
  result.href = uri;

  // set default result target, if specified
  if (options.target) {
    result.target = options.target + '';
  }

  // set default result rel, if specified (#1542)
  if (options.rel) {
    result.rel = options.rel + '';
  }

  // provide control over cross-origin image handling (#2238)
  // https://developer.mozilla.org/en-US/docs/Web/HTML/CORS_enabled_image
  if (options.context === 'image' && options.crossOrigin) {
    result.crossOrigin = options.crossOrigin + '';
  }

  // return
  return result;
}

/**
 * File system loader factory.
 * @param {object} fs - The file system interface.
 * @return {function} - A file loader with the following signature:
 *   param {string} filename - The file system path to load.
 *   param {string} filename - The file system path to load.
 *   return {Promise} A promise that resolves to the file contents.
 */
function fileLoader(fs) {
  return fs
    ? filename => new Promise((accept, reject) => {
        fs.readFile(filename, (error, data) => {
          if (error) reject(error);
          else accept(data);
        });
      })
    : fileReject;
}

/**
 * Default file system loader that simply rejects.
 */
async function fileReject() {
  vegaUtil.error('No file system access.');
}

/**
 * HTTP request handler factory.
 * @param {function} fetch - The Fetch API method.
 * @return {function} - An http loader with the following signature:
 *   param {string} url - The url to request.
 *   param {object} options - An options hash.
 *   return {Promise} - A promise that resolves to the file contents.
 */
function httpLoader(fetch) {
  return fetch
    ? async function(url, options) {
        const opt = vegaUtil.extend({}, this.options.http, options),
              type = options && options.response,
              response = await fetch(url, opt);

        return !response.ok
          ? vegaUtil.error(response.status + '' + response.statusText)
          : vegaUtil.isFunction(response[type]) ? response[type]()
          : response.text();
      }
    : httpReject;
}

/**
 * Default http request handler that simply rejects.
 */
async function httpReject() {
  vegaUtil.error('No HTTP fetch method available.');
}

const isValid = _ => _ != null && _ === _;

const isBoolean = _ => _ === 'true'
  || _ === 'false'
  || _ === true
  || _ === false;

const isDate = _ => !Number.isNaN(Date.parse(_));

const isNumber = _ => !Number.isNaN(+_) && !(_ instanceof Date);

const isInteger = _ => isNumber(_) && Number.isInteger(+_);

const typeParsers = {
  boolean: vegaUtil.toBoolean,
  integer: vegaUtil.toNumber,
  number:  vegaUtil.toNumber,
  date:    vegaUtil.toDate,
  string:  vegaUtil.toString,
  unknown: vegaUtil.identity
};

const typeTests = [
  isBoolean,
  isInteger,
  isNumber,
  isDate
];

const typeList = [
  'boolean',
  'integer',
  'number',
  'date'
];

function inferType(values, field) {
  if (!values || !values.length) return 'unknown';

  const n = values.length,
        m = typeTests.length,
        a = typeTests.map((_, i) => i + 1);

  for (let i = 0, t = 0, j, value; i < n; ++i) {
    value = field ? values[i][field] : values[i];
    for (j = 0; j < m; ++j) {
      if (a[j] && isValid(value) && !typeTests[j](value)) {
        a[j] = 0;
        ++t;
        if (t === typeTests.length) return 'string';
      }
    }
  }

  return typeList[
    a.reduce((u, v) => u === 0 ? v : u, 0) - 1
  ];
}

function inferTypes(data, fields) {
  return fields.reduce((types, field) => {
    types[field] = inferType(data, field);
    return types;
  }, {});
}

function delimitedFormat(delimiter) {
  const parse = function(data, format) {
    const delim = {delimiter: delimiter};
    return dsv(data, format ? vegaUtil.extend(format, delim) : delim);
  };

  parse.responseType = 'text';

  return parse;
}

function dsv(data, format) {
  if (format.header) {
    data = format.header
      .map(vegaUtil.stringValue)
      .join(format.delimiter) + '\n' + data;
  }
  return d3Dsv.dsvFormat(format.delimiter).parse(data + '');
}

dsv.responseType = 'text';

function isBuffer(_) {
  return (typeof Buffer === 'function' && vegaUtil.isFunction(Buffer.isBuffer))
    ? Buffer.isBuffer(_) : false;
}

function json(data, format) {
  const prop = (format && format.property) ? vegaUtil.field(format.property) : vegaUtil.identity;
  return vegaUtil.isObject(data) && !isBuffer(data)
    ? parseJSON(prop(data))
    : prop(JSON.parse(data));
}

json.responseType = 'json';

function parseJSON(data, format) {
  return (format && format.copy)
    ? JSON.parse(JSON.stringify(data))
    : data;
}

const filters = {
  interior: (a, b) => a !== b,
  exterior: (a, b) => a === b
};

function topojson(data, format) {
  let method, object, property, filter;
  data = json(data, format);

  if (format && format.feature) {
    method = topojsonClient.feature;
    property = format.feature;
  } else if (format && format.mesh) {
    method = topojsonClient.mesh;
    property = format.mesh;
    filter = filters[format.filter];
  } else {
    vegaUtil.error('Missing TopoJSON feature or mesh parameter.');
  }

  object = (object = data.objects[property])
    ? method(data, object, filter)
    : vegaUtil.error('Invalid TopoJSON object: ' + property);

  return object && object.features || [object];
}

topojson.responseType = 'json';

const format = {
  dsv: dsv,
  csv: delimitedFormat(','),
  tsv: delimitedFormat('\t'),
  json: json,
  topojson: topojson
};

function formats(name, reader) {
  if (arguments.length > 1) {
    format[name] = reader;
    return this;
  } else {
    return vegaUtil.hasOwnProperty(format, name) ? format[name] : null;
  }
}

function responseType(type) {
  const f = formats(type);
  return f && f.responseType || 'text';
}

function read(data, schema, timeParser, utcParser) {
  schema = schema || {};

  const reader = formats(schema.type || 'json');
  if (!reader) vegaUtil.error('Unknown data format type: ' + schema.type);

  data = reader(data, schema);
  if (schema.parse) parse(data, schema.parse, timeParser, utcParser);

  if (vegaUtil.hasOwnProperty(data, 'columns')) delete data.columns;
  return data;
}

function parse(data, types, timeParser, utcParser) {
  if (!data.length) return; // early exit for empty data

  const locale = vegaFormat.timeFormatDefaultLocale();
  timeParser = timeParser || locale.timeParse;
  utcParser = utcParser || locale.utcParse;

  let fields = data.columns || Object.keys(data[0]),
      datum, field, i, j, n, m;

  if (types === 'auto') types = inferTypes(data, fields);

  fields = Object.keys(types);
  const parsers = fields.map(field => {
    const type = types[field];
    let parts, pattern;

    if (type && (type.startsWith('date:') || type.startsWith('utc:'))) {
      parts = type.split(/:(.+)?/, 2);  // split on first :
      pattern = parts[1];

      if ((pattern[0] === '\'' && pattern[pattern.length-1] === '\'') ||
          (pattern[0] === '"'  && pattern[pattern.length-1] === '"')) {
        pattern = pattern.slice(1, -1);
      }

      const parse = parts[0] === 'utc' ? utcParser : timeParser;
      return parse(pattern);
    }

    if (!typeParsers[type]) {
      throw Error('Illegal format pattern: ' + field + ':' + type);
    }

    return typeParsers[type];
  });

  for (i=0, n=data.length, m=fields.length; i<n; ++i) {
    datum = data[i];
    for (j=0; j<m; ++j) {
      field = fields[j];
      datum[field] = parsers[j](datum[field]);
    }
  }
}

var loader = loaderFactory(
  require('node-fetch'),
  require('fs')
);

exports.format = format;
exports.formats = formats;
exports.inferType = inferType;
exports.inferTypes = inferTypes;
exports.loader = loader;
exports.read = read;
exports.responseType = responseType;
exports.typeParsers = typeParsers;