@kyso/api/publish/utils/get-files.js

const { createHash } = require('crypto')
const flatten = require('arr-flatten')
const unique = require('array-unique')
const { resolve } = require('path')
const ignore = require('ignore')
const _glob = require('glob')
const fs = require('fs-extra')

const hash = async (files) => {
  const map = new Map()
  await Promise.all(
    files.map(async name => {
      const data = await fs.readFile(name)
      const h = createHash('sha1').update(data).digest('hex')
      const entry = map.get(h)
      if (entry) {
        entry.names.push(name)
      } else {
        map.set(createHash('sha1').update(data).digest('hex'), { names: [name], data })
      }
    })
  )
  return map
}

const getFiles = async (dir) => {
  const fileList = await getFilesPaths(dir)
  const hashes = await hash(fileList)

  const files = await Promise.all(Array.prototype.concat.apply([],
    await Promise.all(Array.from(hashes).map(async ([sha, { data, names }]) => // eslint-disable-line
      names.map(async name => ({
        data,
        path: toRelative(name, dir)
      }))
    ))
  ))
  return files
}

const toRelative = (_path, base) => {
  const SEP = process.platform.startsWith('win') ? '\\' : '/'
  const fullBase = base.endsWith(SEP) ? base : base + SEP
  let relative = _path.substr(fullBase.length)

  if (relative.startsWith(SEP)) {
    relative = relative.substr(1)
  }

  return relative.replace(/\\/g, '/')
}

// Base `.gitignore` to which we add entries
// supplied by the user
const IGNORED = `.hg
.git
.kyso
.merge
.gitmodules
.svn
.npmignore
.dockerignore
.gitignore
.jupyter
.local
.cache
.ipython
.ipynb_checkpoints
.*.swp
.DS_Store
.wafpicke-*
.lock-wscript
npm-debug.log
config.gypi
node_modules
CVS
.kyso.json
.kyso-dev.json
Icon?
Icon^M
Icon*`

const glob = async (pattern, options) =>
  new Promise((resolve, reject) => { // eslint-disable-line
    _glob(pattern, options, (error, files) => {
      if (error) {
        reject(error)
      } else {
        resolve(files)
      }
    })
  })


/**
 * Remove leading `./` from the beginning of ignores
 * because our parser doesn't like them :|
 */

const clearRelative = (str) => str.replace(/(\n|^)\.\//g, '$1')

/**
 * Returns the contents of a file if it exists.
 *
 * @return {String} results or `''`
 */

const maybeRead = async (path, default_ = '') => {
  try {
    return await fs.readFile(path, 'utf8')
  } catch (err) {
    return default_
  }
}

/**
 * Transform relative paths into absolutes,
 * and maintains absolutes as such.
 *
 * @param {String} maybe relative path
 * @param {String} parent full path
 */

const asAbsolute = (path, parent) => {
  if (path[0] === '/') {
    return path
  }

  return resolve(parent, path)
}

/**
 * Returns a list of files in the given
 * directory that are subject to be
 * synchronized for kyso.
 *
 * @param {String} full path to directory
 * @param {String} contents of `package.json` to avoid lookup
 * @param {Object} options:
 *  - `limit` {Number|null} byte limit
 *  - `debug` {Boolean} warn upon ignore
 * @return {Array} comprehensive list of paths to sync
 */

async function getFilesPaths(path) {
  // Convert all filenames into absolute paths
  const search = Array.prototype.concat.apply(
    [],
    await Promise.all(
      ['.'].map(file => glob(file, { cwd: path, absolute: true, dot: true }))
    )
  )

  // Compile list of ignored patterns and files
  const kysoIgnore = await maybeRead(resolve(path, '.kysoignore'), null)
  const gitIgnore = kysoIgnore === null
    ? await maybeRead(resolve(path, '.gitignore'))
    : null

  let clr = ''
  if (kysoIgnore || gitIgnore) {
    clr = clearRelative(kysoIgnore === null ? gitIgnore : kysoIgnore)
  }

  const filter = ignore()
    .add(`${IGNORED}\n${clr}`)
    .createFilter()

  const prefixLength = path.length + 1

  // The package.json `files` whitelist still
  // honors kysoignores: https://docs.kysojs.com/files/package.json#files
  // but we don't ignore if the user is explicitly listing files
  // under the kyso namespace, or using files in combination with gitignore
  const overrideIgnores = false
  const accepts = overrideIgnores
    ? () => true
    : file => {
      const relativePath = file.substr(prefixLength)
      if (relativePath === '') {
        return true
      }
      const accepted = filter(relativePath)
      return accepted
    }

  // Locate files
  const files = await explode(search, {
    accepts
  })

  // Get files
  return unique(files)
}

/**
 * Explodes directories into a full list of files.
 * Eg:
 *   in:  ['/a.js', '/b']
 *   out: ['/a.js', '/b/c.js', '/b/d.js']
 *
 * @param {Array} of {String}s representing paths
 * @param {Array} of ignored {String}s.
 * @param {Object} options:
 *  - `limit` {Number|null} byte limit
 *  - `debug` {Boolean} warn upon ignore
 * @return {Array} of {String}s of full paths
 */

async function explode(paths, { accepts }) {
  const list = async file => {
    let path = file
    let s

    if (!accepts(file)) {
      return null
    }

    try {
      s = await fs.stat(path)
    } catch (e) {
      // In case the file comes from `files` or `main`
      // and it wasn't specified with `.js` by the user
      path = `${file}.js`

      try {
        s = await fs.stat(path)
      } catch (e2) {
        return null
      }
    }

    if (s.isDirectory()) {
      const all = await fs.readdir(file)
      /* eslint-disable no-use-before-define */
      return many(all.map(subdir => asAbsolute(subdir, file)))
      /* eslint-enable no-use-before-define */
    } else if (!s.isFile()) {
      return null
    }

    return path
  }

  const many = all => Promise.all(all.map(file => list(file)))
  return flatten(await many(paths)).filter(v => v !== null)
}

module.exports = getFiles