/* Required libraries.  */
/* global g_redis log */
/* eslint consistent-return:warn */
/* eslint no-underscore-dangle: [2, { "allow": ["_id"] }] */
/* eslint no-unused-vars:warn */

const fs = require('fs-extra');
const uuid = require('uuid');

/* Fetch in the other needed components for the this Adaptor */
const { MongoClient } = require('mongodb');

// Other global variables
let adapterDir = '';
let saveFS = false;
let storDir = `${adapterDir}/storage`;
let entityDir = `${adapterDir}/entities`;
let id = null;

const Storage = {
  UNDEFINED: 0,
  DBINFO: 1,
  ADAPTERDB: 2,
  FILESYSTEM: 3,
  IAPDB: 4
};
Object.freeze(Storage);

/* DB UTILS INTERNAL FUNCTIONS         */
/** getFromJson
 * @summary returns information from a json file on the file system
 */
function getFromJson(fileName, filter) {
  const origin = `${id}-dbUtil-getFromJson`;
  log.trace(origin);

  // verify the required data has been provided
  if (fileName === undefined || fileName === null || fileName === '') {
    log.warn(`${origin}: Must provide a file name`);
    return null;
  }

  // get the entity and the action if this is a metric
  let ent = null;
  let act = null;
  let metric = false;
  const useFilter = filter;
  if (useFilter && useFilter.metric) {
    metric = true;
    if (useFilter.metric.entity) {
      ent = useFilter.metric.entity;
    }
    if (useFilter.metric.action) {
      act = useFilter.metric.action;
    }
    if (!ent || !act) {
      log.warn(`${origin}: metric provided with one of entity/action, require both`);
      return null;
    }
    delete useFilter.metric;
  }

  try {
    if (fileName === 'adapter_configs') {
      const retEntity = filter.entity;
      if (!retEntity || !fs.existsSync(`${entityDir}`) || !fs.existsSync(`${entityDir}/${retEntity}`)) {
        log.warn(`${origin}: Could not find adapter entity directory ${retEntity} - nothing to retrieve`);
        return null;
      }
      if (!fs.existsSync(`${entityDir}/${retEntity}/action.json`)) {
        log.warn(`${origin}: Could not find action.json for entity ${retEntity} - nothing to retrieve`);
        return null;
      }

      // load the mockdatafiles
      let files = fs.readdirSync(`${entityDir}/${retEntity}`);
      const mockdatafiles = {};
      if (files.includes('mockdatafiles') && fs.lstatSync(`${entityDir}/${retEntity}/mockdatafiles`).isDirectory()) {
        fs.readdirSync(`${entityDir}/${retEntity}/mockdatafiles`).forEach((file) => {
          if (file.split('.').pop() === 'json') {
            const mockpath = `${entityDir}/${retEntity}/mockdatafiles/${file}`;
            const data = JSON.parse(fs.readFileSync(mockpath));
            mockdatafiles[mockpath.split('/').pop()] = data;
          }
        });
      }

      // load the action data
      let actions;
      if (files.includes('action.json')) {
        actions = JSON.parse(fs.readFileSync(`${entityDir}/${retEntity}/action.json`));
      }

      // Load schema.json and other schemas in remaining json files
      files = files.filter((f) => (f !== 'action.json') && f.endsWith('.json'));
      const schema = [];
      files.forEach((file) => {
        const data = JSON.parse(fs.readFileSync(`${entityDir}/${retEntity}/${file}`));
        schema.push({
          name: file,
          schema: data
        });
      });

      // return the data
      return {
        actions: actions.actions,
        schema,
        mockdatafiles
      };
    }

    // make sure what we need exists and has been provided
    if (!fs.existsSync(`${storDir}`)) {
      log.warn(`${origin}: Could not find adapter storage directory - nothing to retrieve`);
      return null;
    }

    // determine if the file exists so we retrieve the data
    if (fs.existsSync(`${storDir}/${fileName}.json`)) {
      const content = JSON.parse(fs.readFileSync(`${storDir}/${fileName}.json`, 'utf-8'));
      const toReturn = [];

      if (metric) {
        // if metric need to match the entity and action
        content.table.forEach((item) => {
          if (ent === item.entity && act === item.action) {
            toReturn.push(item);
          }
        });
      } else {
        // if not metric must match the items in the filter
        // if no filter, match everything
        let key = [];
        if (useFilter && useFilter.length > 0) {
          key = Object.keys(useFilter);
        }
        content.table.forEach((item) => {
          let push = true;
          // check each key - if it does not match a key do not add to return data
          key.forEach((fil) => {
            if (useFilter[fil] !== item[fil]) push = false;
          });
          if (push) toReturn.push(item);
        });
      }
      // not sure why we do this - a little different than above.
      const filtered = content.table.filter((el) => {
        if (useFilter) {
          Object.keys(useFilter).forEach((obj) => {
            if (el[obj] !== useFilter[obj]) {
              return false;
            }
          });
        }
        return true;
      });
      return toReturn;
    }

    // if no file, nothing to return
    return null;
  } catch (ex) {
    log.warn(`${origin}: Caught Exception ${ex}`);
    return null;
  }
}

/** countJSON
 * @summary returns a count from a json storage file
 */
function countJSON(fileName, filter) {
  const origin = `${id}-dbUtil-countJSON`;
  log.trace(origin);

  // verify the required data has been provided
  if (fileName === undefined || fileName === null || fileName === '') {
    log.warn(`${origin}: Must provide a file name`);
    return null;
  }

  try {
    // make sure what we need exists and has been provided
    if (!fs.existsSync(`${storDir}`)) {
      log.warn(`${origin}: Could not find adapter storage directory - nothing to count`);
      return null;
    }

    // determine if the file exists so we can count data from it
    if (fs.existsSync(`${storDir}/${fileName}.json`)) {
      const data = getFromJson(fileName, filter);
      if (data) {
        return data.length;
      }
      return -1;
    }
  } catch (ex) {
    log.warn(`${origin}: Caught Exception ${ex}`);
    return null;
  }
}

/** saveAsJson
 * @summary saves information into a json storage file
 */
function saveAsJson(fileName, data) {
  const origin = `${id}-dbUtil-saveAsJson`;
  log.trace(origin);

  // verify the required data has been provided
  if (fileName === undefined || fileName === null || fileName === '') {
    log.warn(`${origin}: Must provide a file name`);
    return null;
  }

  // get the entity and the action if this is a metric
  let ent = null;
  let act = null;
  let metric = false;
  const useData = data;
  if (useData && useData.metric) {
    metric = true;
    if (useData.metric.entity) {
      ent = useData.metric.entity;
    }
    if (useData.metric.action) {
      act = useData.metric.action;
    }
    if (!ent || !act) {
      log.warn(`${origin}: metric provided with one of entity/action, require both`);
      return null;
    }
    delete useData.metric;
  }

  try {
    // make sure what we need exists and has been provided
    if (!fs.existsSync(`${storDir}`)) {
      // need to make the storage directory if it does not exist
      fs.mkdirSync(`${storDir}`);
    }

    // determine if the file exists so we add data to it
    if (fs.existsSync(`${storDir}/${fileName}.json`)) {
      // have to read, append & save
      const content = JSON.parse(fs.readFileSync(`${storDir}/${fileName}.json`, 'utf-8'));
      let exists = false;

      content.table.forEach((item) => {
        const toPush = item;
        if (metric) {
          // if it is a metric and we already have entity and action need to edit the data
          if (ent === item.entity && act === item.action) {
            exists = true;
            Object.keys(useData).forEach((key) => {
              if (key === '$inc') {
                Object.keys(useData[key]).forEach((inc) => {
                  if (!toPush[inc]) {
                    toPush[inc] = useData[key][inc];
                  }
                  toPush[inc] += useData[key][inc];
                });
              } else if (key === '$set') {
                Object.keys(useData[key]).forEach((set) => {
                  toPush[set] = useData[key][set];
                });
              }
            });
          }
        }
      });
      if (!exists) {
        // push new thing to table.
        const toPush = {};
        const keysArray = Object.keys(useData);
        keysArray.forEach((i) => {
          const newKeys = Object.keys(useData[i]);
          newKeys.forEach((j) => {
            toPush[j] = useData[i][j];
          });
        });
        content.table.push(toPush);
      }

      // now that is updated, write the file back out
      fs.writeFileSync(`${storDir}/${fileName}.json`, JSON.stringify(content, null, 2));
      return useData;
    }

    // if the file has not been created yet
    const obj = { table: [] };
    const toPush = {};
    const keysArray = Object.keys(data);
    keysArray.forEach((outer) => {
      const newKeys = Object.keys(data[outer]);
      newKeys.forEach((inner) => {
        toPush[inner] = data[outer][inner];
      });
    });
    obj.table.push(toPush);

    // write the file out to the file system
    fs.writeFileSync(`${storDir}/${fileName}.json`, JSON.stringify(obj, null, 2));
    return data;
  } catch (ex) {
    log.warn(`${origin}: Caught Exception ${ex}`);
    return null;
  }
}

/** removeFromJSON
 * @summary removes information from a json storage file
 */
function removeFromJSON(fileName, filter, multiple) {
  const origin = `${id}-dbUtil-removeFromJSON`;
  log.trace(origin);

  // verify the required data has been provided
  if (fileName === undefined || fileName === null || fileName === '') {
    log.warn(`${origin}: Must provide a file name`);
    return null;
  }

  // get the entity and the action if this is a metric
  let ent = null;
  let act = null;
  let metric = false;
  const useFilter = filter;
  if (useFilter && useFilter.metric) {
    metric = true;
    if (useFilter.metric.entity) {
      ent = useFilter.metric.entity;
    }
    if (useFilter.metric.action) {
      act = useFilter.metric.action;
    }
    if (!ent && !act) {
      log.warn(`${origin}: metric provided with one of entity/action, require both`);
      return null;
    }
    delete useFilter.metric;
  }

  try {
    // make sure what we need exists and has been provided
    if (!fs.existsSync(`${storDir}`)) {
      log.warn(`${origin}: Could not find adapter storage directory - nothing to remove`);
      return null;
    }

    // determine if the file exists so we remove data from it
    if (fs.existsSync(`${storDir}/${fileName}.json`)) {
      const content = JSON.parse(fs.readFileSync(`${storDir}/${fileName}.json`, 'utf-8'));
      const toReturn = [];

      // if this is a metric make sure the entity and action match
      if (metric) {
        content.table = content.table.filter((item) => {
          if (ent === item.entity && act === item.action) {
            toReturn.push(item);
            return false;
          }
          return true;
        });
      } else {
        // go through content to determine if item matches the filter
        let ctr = 0;
        // create the contents that are being removed
        const removed = content.table.filter((el) => {
          Object.keys(useFilter).forEach((obj) => {
            if (el[obj] !== useFilter[obj]) {
              return false;
            }
          });
          ctr += 1;
          if (!multiple && ctr > 1) {
            return false;
          }
          return true;
        });
        let ctr1 = 0;
        // remove the items from the contents
        content.table = content.table.filter((el, i) => {
          Object.keys(useFilter).forEach((obj) => {
            if (el[obj] !== useFilter[obj]) {
              return true;
            }
          });
          ctr1 += 1;
          if (!multiple && ctr1 > 1) {
            return true;
          }
          return false;
        });

        // write the contents back to the file
        fs.writeFileSync(`${storDir}/${fileName}.json`, JSON.stringify(content, null, 2));
        return removed;
      }

      // write the contents back to the file
      fs.writeFileSync(`${storDir}/${fileName}.json`, JSON.stringify(content, null, 2));
      return toReturn;
    }

    log.error(`${origin}: Collection ${fileName} does not exist`);
    return null;
  } catch (ex) {
    log.warn(`${origin}: Caught Exception ${ex}`);
    return null;
  }
}

/** deleteJSON
 * @summary deletes a json storage file
 */
function deleteJSON(fileName) {
  const origin = `${id}-dbUtil-deleteJSON`;
  log.trace(origin);

  // verify the required data has been provided
  if (fileName === undefined || fileName === null || fileName === '') {
    log.warn(`${origin}: Must provide a file name`);
    return null;
  }

  try {
    // make sure what we need exists and has been provided
    if (!fs.existsSync(`${storDir}`)) {
      log.warn(`${origin}: Could not find adapter storage directory - nothing to delete`);
      return null;
    }

    // determine if the file exists so we remove - also assume we add a .json suffix
    if (fs.existsSync(`${storDir}/${fileName}.json`)) {
      fs.remove(`${storDir}/${fileName}.json`).catch((some) => {
        log.info(`${origin}: ${some}`);
        fs.rmdirSync(`${storDir}/${fileName}.json`);
      });
    }

    // successful -- return the fileName
    return fileName;
  } catch (ex) {
    log.warn(`${origin}: Caught Exception ${ex}`);
    return null;
  }
}

class DBUtil {
  /**
   * These are database utilities that can be used by the adapter to interact with a
   * mongo database
   * @constructor
   */
  constructor(prongId, properties, directory) {
    this.myid = prongId;
    id = prongId;
    this.baseDir = directory;
    adapterDir = this.baseDir;
    storDir = `${adapterDir}/storage`;
    entityDir = `${adapterDir}/entities`;
    this.props = properties;
    this.adapterMongoClient = null;

    // set up the properties I care about
    this.refreshProperties(properties);
  }

  /**
   * refreshProperties is used to set up all of the properties for the db utils.
   * It allows properties to be changed later by simply calling refreshProperties rather
   * than having to restart the db utils.
   *
   * @function refreshProperties
   * @param {Object} properties - an object containing all of the properties
   */
  refreshProperties(properties) {
    const origin = `${this.myid}-dbUtil-refreshProperties`;
    log.trace(origin);
    this.dburl = null;
    this.dboptions = {};
    this.database = this.myid;

    // verify the necessary information was received
    if (!properties) {
      log.error(`${origin}: DB Utils received no properties!`);
      return;
    }
    if (!properties.mongo || !properties.mongo.host) {
      log.info(`${origin}: No default adapter database configured!`);
      return;
    }

    // set the database port
    let port = 27017;
    if (properties.mongo.port) {
      port = properties.mongo.port;
    }
    // set the database
    if (properties.mongo.database) {
      this.database = properties.mongo.database;
    }
    // set the user
    let username = null;
    if (properties.mongo.username) {
      username = properties.mongo.username;
    }
    // set the password
    let password = null;
    if (properties.mongo.password) {
      password = properties.mongo.password;
    }

    // format the database url
    this.dburl = 'mongodb://';
    log.info(`${origin}: Default adapter database at: ${properties.mongo.host}`);

    if (username) {
      this.dburl += `${encodeURIComponent(username)}:${encodeURIComponent(password)}@`;
      log.info(`${origin}: Default adapter database will use authentication.`);
    }
    this.dburl += `${encodeURIComponent(properties.mongo.host)}:${encodeURIComponent(port)}/${encodeURIComponent(this.database)}`;

    // are we using a replication set need to add it to the url
    if (properties.mongo.replSet) {
      this.dburl += `?${properties.mongo.replSet}`;
      log.info(`${origin}: Default adapter database will use replica set.`);
    }

    // Do we need SSL to connect to the database
    if (properties.mongo.db_ssl && properties.mongo.db_ssl.enabled === true) {
      log.info(`${origin}: Default adapter database will use SSL.`);
      this.dboptions.ssl = true;

      // validate the server's certificate against a known certificate authority?
      if (properties.mongo.db_ssl.accept_invalid_cert === false) {
        this.dboptions.sslValidate = true;
        log.info(`${origin}: Default adapter database will use Certificate based SSL.`);
        // if validation is enabled, we need to read the CA file
        if (properties.mongo.db_ssl.ca_file) {
          try {
            this.dboptions.sslCA = [fs.readFileSync(properties.mongo.db_ssl.ca_file)];
          } catch (err) {
            log.error(`${origin}: Error: unable to load Mongo CA file: ${err}`);
          }
        } else {
          log.error(`${origin}: Error: Certificate validation is enabled but a CA is not specified.`);
        }
        if (properties.mongo.db_ssl.key_file) {
          try {
            this.dboptions.sslKey = [fs.readFileSync(properties.mongo.db_ssl.key_file)];
          } catch (err) {
            log.error(`${origin}: Error: Unable to load Mongo Key file: ${err}`);
          }
        }
        if (properties.mongo.db_ssl.cert_file) {
          try {
            this.dboptions.sslCert = [fs.readFileSync(properties.mongo.db_ssl.cert_file)];
          } catch (err) {
            log.error(`${origin}: Error: Unable to load Mongo Certificate file: ${err}`);
          }
        }
      } else {
        this.dboptions.sslValidate = false;
        log.info(`${origin}: Default adapter database not using Certificate based SSL.`);
      }
    } else {
      log.info(`${origin}: Default adapter database not using SSL.`);
    }

    // if we were provided with a path to save metrics - instead of just a boolean
    // will assume this is a place where all adapter stuff can go!
    saveFS = this.props.save_metric || false;
    if (saveFS && typeof saveFS === 'string' && saveFS !== '') {
      storDir = saveFS;
    }
  }

  /**
   * Call to determine the storage target. If the storage is a Mongo database, then
   * the client connection and database is returned.
   *
   * @function determineStorage
   * @param {object} dbInfo - the url for the database to connect to (dburl, dboptions, database)
   * @param {callback} callback - the error, the storage, the Mongo client connection, the Mongo database
   */
  determineStorage(dbInfo, callback) {
    const origin = `${this.myid}-dbUtil-determineStorage`;

    if (dbInfo) {
      // priority 1 - use the dbInfo passed in
      if (dbInfo.dburl && dbInfo.database) {
        MongoClient.connect(dbInfo.dburl, dbInfo.dboptions, (err, mongoClient) => {
          if (err) {
            log.error(`${origin}: Error! Failed to connect to database: ${err}`);
            return callback(err, null, null, null);
          }
          log.debug('using dbinfo');
          return callback(null, Storage.DBINFO, mongoClient, dbInfo.database);
        });
      } else {
        const err = 'Error! Marlformed dbInfo';
        return callback(err, null, null, Storage.DBINFO);
      }
    } else if (this.adapterMongoClient) {
      // priority 2 - use the adapter database
      log.debug('using adapter db');
      return callback(null, Storage.ADAPTERDB, this.adapterMongoClient, this.database);
    } else if (this.dburl === null && dbInfo === null) {
      // priority 3 - use the filesystem
      log.debug('using filesystem');
      return callback(null, Storage.FILESYSTEM, null, null);
    }
  }

  /**
   * Call to connect and authenticate to the adapter database
   *
   * @function connect
   */
  connect(callback) {
    // callback format: (this.alive, mongoClient); generally mongoClient should only be used if this.alive is true.
    const origin = `${this.myid}-dbUtil-connect`;
    log.trace(origin);

    // const options = (replSetEnabled === true) ? { replSet: opts } : { server: opts };
    log.debug(`${origin}: Connecting to MongoDB with options ${JSON.stringify(this.dboptions)}`);

    // Now we will start the process of connecting to mongo db
    return MongoClient.connect(this.dburl, this.dboptions, (err, mongoClient) => {
      if (!mongoClient) {
        log.error(`${origin}: Error! Exiting... Must start MongoDB first ${err}`);
        this.alive = false;
        return callback(this.alive);
      }
      mongoClient.on('close', () => {
        this.alive = false;
        log.error(`${origin}: MONGO CONNECTION LOST...`);
      });

      mongoClient.on('reconnect', () => {
        // we still need to check if we are properly authenticated
        // so we just list collections to test it.
        this.clientDB.listCollections().toArray((error) => {
          if (error) {
            log.error(`${origin}: ${error}`);
            this.alive = false;
          } else {
            log.info(`${origin}: MONGO CONNECTION BACK...`);
            this.alive = true;
            this.adapterMongoClient = mongoClient;
          }
        });
      });

      mongoClient.db(this.database).serverConfig.on('left', (type) => {
        if (type === 'primary') {
          this.alive = false;
          log.info(`${origin}: MONGO PRIMARY CONNECTION LOST...`);
        } else if (type === 'secondary') {
          log.info(`${origin}: MONGO SECONDARY CONNECTION LOST...`);
        }
      });

      mongoClient.db(this.database).serverConfig.on('joined', (type) => {
        if (type === 'primary') {
          log.info(`${origin}: MONGO PRIMARY CONNECTION BACK...`);
        } else if (type === 'secondary') {
          log.info(`${origin}: MONGO SECONDARY CONNECTION BACK...`);
        }
      });

      log.info(`${origin}: mongo running @${this.dburl}/${this.database}`);
      this.clientDB = mongoClient.db(this.database);
      this.adapterMongoClient = mongoClient;

      // we don't have authentication defined but we still need to check if Mongo does not
      // require one, so we just list collections to test if it's doable.
      return this.clientDB.listCollections().toArray((error) => {
        if (error) {
          log.error(`${origin}: ${error}`);
          this.alive = false;
        } else {
          log.info(`${origin}: MongoDB connection has been established`);
          this.alive = true;
          this.adapterMongoClient = mongoClient;
        }
        return callback(this.alive, mongoClient);
      });
    });
  }

  /**
   * Call to disconnect from the adapter database
   *
   * @function disconnect
   */
  disconnect() {
    if (this.adapterMongoClient) {
      this.adapterMongoClient.close();
    }
  }

  /**
   * createCollection creates the provided collection in the file system or database.
   *
   * @function createCollection
   * @param {string} collectionName - the name of the collection to create
   * @param {object} dbInfo - the url for the database to connect to (dburl, dboptions, database)
   * @param {boolean} fsWrite - turn on write to the file system
   */
  createCollection(collectionName, dbInfo, fsWrite, callback) {
    const origin = `${this.myid}-dbUtil-createCollection`;
    log.trace(origin);

    try {
      // verify the required data has been provided
      if (!collectionName || (typeof collectionName !== 'string')) {
        log.warn(`${origin}: Missing Collection Name or not string`);
        return callback(`${origin}: Missing Collection Name or not string`, null);
      }

      return this.determineStorage(dbInfo, (storageError, storage, mongoClient, database) => {
        if (storageError) {
          return callback(`${origin}: ${storageError}`, null);
        }

        // if using file storage
        if (storage === Storage.FILESYSTEM) {
          // if there is no adapter directory - can not do anything so error
          if (!fs.existsSync(`${this.baseDir}`)) {
            log.warn(`${origin}: Not able to create storage - missing base directory!`);
            return callback(`${origin}: Not able to create storage - missing base directory!`, null);
          }
          // if there is no storage directory - create it
          if (!fs.existsSync(`${this.baseDir}/storage`)) {
            fs.mkdirSync(`${this.baseDir}/storage`);
          }
          // if the collection already exists - no need to create it
          if (fs.existsSync(`${this.baseDir}/storage/${collectionName}`)) {
            log.debug(`${origin}: storage file collection already exists`);
            return callback(null, collectionName);
          }
          // create the new collection on the file system
          fs.mkdirSync(`${this.baseDir}/storage/${collectionName}`);
          log.debug(`${origin}: storage file collection ${collectionName} created`);
          return callback(null, collectionName);
        }

        // if using MongoDB storage
        if (storage === Storage.DBINFO || storage === Storage.ADAPTERDB) {
          return mongoClient.db(database).createCollection(collectionName, (err, res) => {
            if (storage === Storage.DBINFO && mongoClient) mongoClient.close();
            if (err) {
              // error we get back if the collection already existed - not a true error
              if (err.codeName === 'NamespaceExists') {
                log.debug(`${origin}: database collection already exists`);
                return callback(null, collectionName);
              }
              log.warn(`${origin}: Error creating collection ${err}`);
              return callback(`${origin}: Error creating collection ${err}`, null);
            }
            log.spam(`${origin}: db response ${res}`);
            log.debug(`${origin}: database collection ${collectionName} created`);
            return callback(null, collectionName);
          });
        }
      });
    } catch (ex) {
      log.warn(`${origin}: Caught Exception - ${ex}`);
      return callback(`${origin}: Caught Exception - ${ex}`, null);
    }
  }

  /**
   * removeCollection removes the provided collection from the file system or database.
   *
   * @function removeCollection
   * @param {string} collectionName - the name of the collection to remove
   * @param {object} dbInfo - the url for the database to connect to (dburl, dboptions, database)
   * @param {boolean} fsWrite - turn on write to the file system
   */
  removeCollection(collectionName, dbInfo, fsWrite, callback) {
    const origin = `${this.myid}-dbUtil-removeCollection`;
    log.trace(origin);

    try {
      // verify the required data has been provided
      if (!collectionName || (typeof collectionName !== 'string')) {
        log.warn(`${origin}: Missing Collection Name or not string`);
        return callback(`${origin}: Missing Collection Name or not string`, null);
      }

      return this.determineStorage(dbInfo, (storageError, storage, mongoClient, database) => {
        if (storageError) {
          return callback(`${origin}: ${storageError}`, null);
        }

        // if using file storage
        if (storage === Storage.FILESYSTEM) {
          const deld = deleteJSON(collectionName);
          if (deld) {
            log.debug(`${origin}: storage file collection ${collectionName} removed`);
            return callback(null, deld);
          }
          log.debug(`${origin}: could not remove storage file collection`);
          return callback(`${origin}: could not remove storage file collection`, null);
        }

        // if using MongoDB storage
        if (storage === Storage.DBINFO || storage === Storage.ADAPTERDB) {
          // get the list of collections from the database - can we just drop?
          return mongoClient.db(database).listCollections().toArray((err, result) => {
            if (err) {
              log.warn(`${origin}: Failed to get collections ${err}`);
              return callback(`${origin}: Failed to get collections ${err}`, null);
            }
            // go through the collections to get the correct one for removal
            for (let e = 0; e < result.length; e += 1) {
              if (result[e].name === collectionName) {
                // now that we found it, remove it
                return mongoClient.db(database).collection(collectionName).drop({}, (err1, res) => {
                  if (storage === Storage.DBINFO && mongoClient) mongoClient.close();
                  if (err1) {
                    log.warn(`${origin}: Failed to remove collection ${err1}`);
                    return callback(`${origin}: Failed to remove collection ${err1}`, null);
                  }
                  log.spam(`${origin}: db response ${res}`);
                  log.debug(`${origin}: database collection ${collectionName} removed`);
                  return callback(null, collectionName);
                });
              }
            }
            return callback(null, collectionName);
          });
        }
      });
    } catch (ex) {
      log.warn(`${origin}: Caught Exception - ${ex}`);
      return callback(`${origin}: Caught Exception - ${ex}`, null);
    }
  }

  /**
   * Call to create an item in the database
   *
   * @function create
   * @param {string} collectionName - the collection to save the item in. (required)
   * @param {string} data - the modification to make. (required)
   * @param {object} dbInfo - the url for the database to connect to (dburl, dboptions, database)
   * @param {boolean} fsWrite - turn on write to the file system
   */
  create(collectionName, data, dbInfo, fsWrite, callback) {
    const origin = `${this.myid}-dbUtil-create`;
    log.trace(origin);

    try {
      // verify the required data has been provided
      if (!collectionName) {
        log.warn(`${origin}: Missing Collection Name`);
        return callback(`${origin}: Missing Collection Name`, null);
      }
      if (!data) {
        log.warn(`${origin}: Missing data to add`);
        return callback(`${origin}: Missing data to add`, null);
      }
      if ((data.entity && !data.action) || (!data.entity && data.action)) {
        log.warn(`${origin}: Inconsistent entity/action set`);
        return callback(`${origin}: Inconsistent entity/action set`, null);
      }

      // create the unique identifier (should we let mongo do this?)
      const dataInfo = data;
      if (!{}.hasOwnProperty.call(dataInfo, '_id')) {
        dataInfo._id = uuid.v4();
      }

      return this.determineStorage(dbInfo, (storageError, storage, mongoClient, database) => {
        if (storageError) {
          return callback(`${origin}: ${storageError}`, null);
        }

        // if using file storage
        if (storage === Storage.FILESYSTEM) {
          // save it to file in the adapter storage directory
          const saved = saveAsJson(collectionName, data);
          if (!saved) {
            log.warn(`${origin}: Data has not been saved to file storage`);
            return callback(`${origin}: Data has not been saved to file storage`, null);
          }
          log.debug(`${origin}: Data saved in file storage`);
          return callback(null, saved);
        }

        // if using MongoDB storage
        if (storage === Storage.DBINFO || storage === Storage.ADAPTERDB) {
          // Add the data to the database
          // insertOne has only 2 parameters: the data to be added & callback. Not an identifier.
          return mongoClient.db(database).collection(collectionName).insertOne(dataInfo, (err, result) => {
            if (storage === Storage.DBINFO && mongoClient) mongoClient.close();
            if (err) {
              log.warn(`${origin}: Failed to insert data in collection ${err}`);
              return callback(`${origin}: Failed to insert data in collection ${err}`, null);
            }
            log.spam(`${origin}: db response ${result}`);
            log.debug(`${origin}: Data saved in database`);
            return callback(null, data);
          });
        }
      });
    } catch (ex) {
      log.warn(`${origin}: Caught Exception - ${ex}`);
      return callback(`${origin}: Caught Exception - ${ex}`, null);
    }
  }

  /**
   * Call to create an index in the database
   *
   * @function createIndex
   * @param {string} collectionName - the collection to index. (required)
   * @param {string} fieldOrSpec - what to index. (required)
   * @param {object} dbInfo - the url for the database to connect to (dburl, dboptions, database)
   */
  createIndex(collectionName, fieldOrSpec, options, dbInfo, callback) {
    const origin = `${this.myid}-dbUtil-createIndex`;
    log.trace(origin);

    try {
      // verify the required data has been provided
      if (!collectionName) {
        log.warn(`${origin}: Missing Collection Name`);
        return callback(`${origin}: Missing Collection Name`, null);
      }
      if (!fieldOrSpec) {
        log.warn(`${origin}: Missing Specs`);
        return callback(`${origin}: Missing Specs`, null);
      }

      return this.determineStorage(dbInfo, (storageError, storage, mongoClient, database) => {
        if (storageError) {
          return callback(`${origin}: ${storageError}`, null);
        }

        // if using file storage
        if (storage === Storage.FILESYSTEM) {
          // no database - no index
          log.warn(`${origin}: No database - no index`);
          return callback(`${origin}: No database - no index`, null);
        }

        // if using MongoDB storage
        if (storage === Storage.DBINFO || storage === Storage.ADAPTERDB) {
          // create the index on the collection
          return mongoClient.db(database).collection(collectionName).createIndex(fieldOrSpec, options || {}, (err, res) => {
            if (storage === Storage.DBINFO && mongoClient) mongoClient.close();
            if (err) {
              log.warn(`${origin}: Failed to index data in collection ${err}`);
              return callback(`${origin}: Failed to index data in collection ${err}`, null);
            }
            log.debug(`${origin}: Data in collection indexed`);
            return callback(null, res);
          });
        }
      });
    } catch (ex) {
      log.warn(`${origin}: Caught Exception - ${ex}`);
      return callback(`${origin}: Caught Exception - ${ex}`, null);
    }
  }

  /**
   * Call to count the documents in a collection
   *
   * @function countDocuments
   * @param {string} collectionName - the collection to count documents in. (required)
   * @param {object} query - the query to minimize documents you count. (required)
   * @param {object} dbInfo - the url for the database to connect to (dburl, dboptions, database)
   * @param {boolean} fsWrite - turn on write to the file system
   */
  countDocuments(collectionName, query, options, dbInfo, fsWrite, callback) {
    const origin = `${this.myid}-dbUtil-countDocuments`;
    log.trace(origin);

    try {
      // verify the required data has been provided
      if (!collectionName) {
        log.warn(`${origin}: Missing Collection Name`);
        return callback(`${origin}: Missing Collection Name`, null);
      }

      return this.determineStorage(dbInfo, (storageError, storage, mongoClient, database) => {
        if (storageError) {
          return callback(`${origin}: ${storageError}`, null);
        }

        // if using file storage
        if (storage === Storage.FILESYSTEM) {
          // get a count from the JSON
          const data = countJSON(collectionName, query);
          if (!data || data === -1) {
            log.warn(`${origin}: Could not count data from file storage`);
            return callback(`${origin}: Could not count data from file storage`, null);
          }
          log.debug(`${origin}: Count from file storage ${data}`);
          return callback(null, data);
        }

        // if using MongoDB storage
        if (storage === Storage.DBINFO || storage === Storage.ADAPTERDB) {
          // get the count from mongo
          return mongoClient.db(database).collection(collectionName).count(query, options, (err, res) => {
            if (storage === Storage.DBINFO && mongoClient) mongoClient.close();
            if (err) {
              log.warn(`${origin}: Failed to count collection ${err}`);
              return callback(`${origin}: Failed to count collection ${err}`, null);
            }
            log.debug(`${origin}: Count from database ${res}`);
            return callback(null, res);
          });
        }
      });
    } catch (ex) {
      log.warn(`${origin}: Caught Exception - ${ex}`);
      return callback(`${origin}: Caught Exception - ${ex}`, null);
    }
  }

  /**
   * Delete items from a collection
   *
   * @function delete
   * @param {string} collectionName - the collection to remove document from. (required)
   * @param {object} filter - the filter for the document(s) to remove. (required)
   * @param {object} dbInfo - the url for the database to connect to (dburl, dboptions, database)
   * @param {boolean} fsWrite - turn on write to the file system
   */
  delete(collectionName, filter, options, multiple, dbInfo, fsWrite, callback) {
    const origin = `${this.myid}-dbUtil-delete`;
    log.trace(origin);

    try {
      // verify the required data has been provided
      if (!collectionName) {
        log.warn(`${origin}: Missing Collection Name`);
        return callback(`${origin}: Missing Collection Name`, null);
      }
      if (!filter) {
        log.warn(`${origin}: Missing Filter`);
        return callback(`${origin}: Missing Filter`, null);
      }
      if (multiple === undefined || multiple === null) {
        log.warn(`${origin}: Missing Multiple flag`);
        return callback(`${origin}: Missing Multiple flag`, null);
      }

      return this.determineStorage(dbInfo, (storageError, storage, mongoClient, database) => {
        if (storageError) {
          return callback(`${origin}: ${storageError}`, null);
        }

        // if using file storage
        if (storage === Storage.FILESYSTEM) {
          // verify the collection exists
          if (!fs.existsSync(`${adapterDir}/storage/${collectionName}.json`)) {
            log.warn(`${origin}: Collection ${collectionName} does not exist`);
            return callback(null, `${origin}: Collection ${collectionName} does not exist`);
          }
          // remove the item from the collection
          const deld = removeFromJSON(collectionName, filter, multiple);
          if (!deld) {
            log.warn(`${origin}: Data has not been deleted from file storage`);
            return callback(`${origin}: Data has not been deleted from file storage`, null);
          }
          log.debug(`${origin}: Data has been deleted from file storage`);
          return callback(null, deld);
        }

        // if using MongoDB storage
        if (storage === Storage.DBINFO || storage === Storage.ADAPTERDB) {
          if (!multiple) {
            // delete the single item from mongo
            return mongoClient.db(database).collection(collectionName).deleteOne(filter, options, (err, res) => {
              if (storage === Storage.DBINFO && mongoClient) mongoClient.close();
              if (err) {
                log.warn(`${origin}: Failed to delete data from database ${err}`);
                return callback(`${origin}: Failed delete data from database ${err}`, null);
              }
              log.debug(`${origin}: Data has been deleted from database`);
              return callback(null, res);
            });
          }

          // delete the multiple items from mongo
          return mongoClient.db(database).collection(collectionName).deleteMany(filter, options, (err, res) => {
            if (storage === Storage.DBINFO && mongoClient) mongoClient.close();
            if (err) {
              log.warn(`${origin}: Failed to delete data from database ${err}`);
              return callback(`${origin}: Failed delete data from database ${err}`, null);
            }
            log.debug(`${origin}: Data has been deleted from database`);
            return callback(null, res);
          });
        }
      });
    } catch (ex) {
      log.warn(`${origin}: Caught Exception - ${ex}`);
      return callback(`${origin}: Caught Exception - ${ex}`, null);
    }
  }

  /**
   * Replace an item in a collection
   *
   * @function replaceOne
   * @param {string} collectionName - the collection to replace document in. (required)
   * @param {object} filter - the filter for the document(s) to replace. (required)
   * @param {object} doc - the filter for the document(s) to replace. (required)
   * @param {object} dbInfo - the url for the database to connect to (dburl, dboptions, database)
   * @param {boolean} fsWrite - turn on write to the file system
   */
  replaceOne(collectionName, filter, doc, options, dbInfo, fsWrite, callback) {
    const origin = `${this.myid}-dbUtil-replaceOne`;
    log.trace(origin);

    try {
      // verify the required data has been provided
      if (!collectionName) {
        log.warn(`${origin}: Missing Collection Name`);
        return callback(`${origin}: Missing Collection Name`, null);
      }
      if (!filter) {
        log.warn(`${origin}: Missing Filter`);
        return callback(`${origin}: Missing Filter`, null);
      }
      if (!doc) {
        log.warn(`${origin}: Missing Document`);
        return callback(`${origin}: Missing Document`, null);
      }

      return this.determineStorage(dbInfo, (storageError, storage, mongoClient, database) => {
        if (storageError) {
          return callback(`${origin}: ${storageError}`, null);
        }

        // if using file storage
        if (storage === Storage.FILESYSTEM) {
          // remove the data from the collection
          const rem = removeFromJSON(collectionName, filter, false);
          if (rem) {
            // add the data into the collection
            const sav = saveAsJson(collectionName, doc);
            if (sav) {
              log.debug(`${origin}: Data replaced in file storage`);
              return callback(null, sav);
            }
            log.warn(`${origin}: Could not save doc into file storage`);
            return callback(`${origin}: Could not save doc into file storage`, null);
          }
          log.warn(`${origin}: Could not delete from file storage`);
          return callback(`${origin}: Could not delete from file storage`, null);
        }

        // if using MongoDB storage
        if (storage === Storage.DBINFO || storage === Storage.ADAPTERDB) {
          // replace an items in mongo
          return mongoClient.db(database).collection(collectionName).replaceOne(filter, doc, options, (err, res) => {
            if (storage === Storage.DBINFO && mongoClient) mongoClient.close();
            if (err) {
              log.warn(`${origin}: Failed to replace data in database ${err}`);
              return callback(`${origin}: Failed replace data in database ${err}`, null);
            }
            log.debug(`${origin}: Data replaced in file storage`);
            return callback(null, res);
          });
        }
      });
    } catch (ex) {
      log.warn(`${origin}: Caught Exception - ${ex}`);
      return callback(`${origin}: Caught Exception - ${ex}`, null);
    }
  }

  /**
   * Call to find items in the database
   *
   * @function find
   * @param {string} collectionName - the collection name to search in. (required)
   * @param {object} options - the options to use to find data.
   * options = {
   *  filter : <filter Obj>,
   *  sort : <sort Obj>,
   *  start : <start position>,
   *  limit : <limit of results>,
   * }
   * @param {object} dbInfo - the url for the database to connect to (dburl, dboptions, database)
   * @param {boolean} fsWrite - turn on write to the file system
   */
  find(collectionName, options, dbInfo, fsWrite, callback) {
    const origin = `${this.myid}-dbUtil-find`;
    log.trace(origin);

    try {
      // verify the required data has been provided
      if (!collectionName) {
        log.warn(`${origin}: Missing Collection Name`);
        return callback(`${origin}: Missing Collection Name`, null);
      }

      // get the collection so we can run the remove on the collection
      let filter = {};
      let start = 0;
      let sort = {};
      let limit = 10;
      if (options) {
        filter = options.filter || {};
        start = options.start || 0;
        sort = options.sort || {};
        if (Object.hasOwnProperty.call(options, 'limit')) {
          ({ limit } = options);
        }
      }

      // If limit is not specified, default to 10.
      // Note: limit may be 0, which is equivalent to setting no limit.

      // Replace filter with regex to allow for substring lookup
      // TODO: Need to create a new filter object instead of mutating the exsisting one
      const filterKeys = Object.keys(filter).filter((key) => (key[0] !== '$' && typeof filter[key] === 'string'));
      filterKeys.map((key) => {
        try {
          const escapedFilter = filter[key].replace(/([.?*+^$[\]\\(){}|-])/g, '\\$1');
          const regexedFilter = new RegExp(`.*${escapedFilter}.*`, 'i');
          filter[key] = {
            $regex: regexedFilter
          };
        } catch (e) {
          delete filter[key];
        }
        return key;
      });

      return this.determineStorage(dbInfo, (storageError, storage, mongoClient, database) => {
        if (storageError) {
          return callback(`${origin}: ${storageError}`, null);
        }

        // if using file storage
        if (storage === Storage.FILESYSTEM) {
          // Find it from file in the adapter
          let toReturn = getFromJson(collectionName, options);
          if (collectionName === 'adapter_configs') {
            log.debug(`${origin}: Data retrieved from file storage`);
            return callback(null, [toReturn]);
          }
          if (toReturn && toReturn.length > limit) {
            let curEnd = start + limit;
            if (curEnd < toReturn.length) {
              curEnd = toReturn.length;
            }
            toReturn = toReturn.slice(start, curEnd);
          }
          log.debug(`${origin}: Data retrieved from file storage`);
          return callback(null, toReturn);
        }

        // if using MongoDB storage
        if (storage === Storage.DBINFO || storage === Storage.ADAPTERDB) {
          // Find the data in the database
          return mongoClient.db(database).collection(collectionName).find(filter).sort(sort)
            .skip(start)
            .limit(limit)
            .toArray()
            .then((value) => {
              log.debug(`${origin}: Data retrieved from database`);
              return callback(null, value);
            })
            .finally(() => {
              if (storage === Storage.DBINFO && mongoClient) mongoClient.close();
            });
        }
      });
    } catch (ex) {
      log.warn(`${origin}: Caught Exception - ${ex}`);
      return callback(`${origin}: Caught Exception - ${ex}`, null);
    }
  }

  /**
   * Call to find an item in a collection and modify it.
   *
   * @function findAndModify
   * @param {string} collectionName - the collection to find things from. (required)
   * @param {object} filter - the filter used to find objects. (optional)
   * @param {array} sort - how to sort the items (first one in order will be modified). (optional)
   * @param {object} data - the modification to make. (required)
   * @param {boolean} upsert - option for the whether to insert new objects. (optional)
   * @param {object} dbInfo - the url for the database to connect to (dburl, dboptions, database) (optional)
   * @param {boolean} fsWrite - turn on write to the file system (optional)
   * @param {updateCallback} callback - a callback function to return a result
   *                                    (the new object) or the error
   */
  findAndModify(collectionName, filter, sort, data, upsert, dbInfo, fsWrite, callback) {
    const origin = `${this.myid}-dbUtil-findAndModify`;
    log.trace(origin);

    try {
      // verify the required data has been provided
      if (collectionName === undefined || collectionName === null || collectionName === '') {
        log.warn(`${origin}: Missing Collection Name`);
        return callback(`${origin}: Missing Collection Name`, null);
      }
      if (data === undefined || data === null || typeof data !== 'object' || Object.keys(data).length === 0) {
        log.warn(`${origin}: Missing data for modification`);
        return callback(`${origin}: Missing data for modification`, null);
      }

      const options = {
        sort,
        upsert,
        returnOriginal: false
      };

      return this.determineStorage(dbInfo, (storageError, storage, mongoClient, database) => {
        if (storageError) {
          return callback(`${origin}: ${storageError}`, null);
        }

        // if using file storage
        if (storage === Storage.FILESYSTEM) {
          // save it to file in the adapter storage directory
          const saved = saveAsJson(collectionName, data);
          if (!saved) {
            log.error(`${origin}: Data has not been saved`);
            return callback(`${origin}: Data has not been saved`, null);
          }
          log.debug(`${origin}: Data modified in file storage`);
          return callback(null, saved);
        }

        // if using MongoDB storage
        if (storage === Storage.DBINFO || storage === Storage.ADAPTERDB) {
          // find and modify the data in the database
          return mongoClient.db(database).collection(collectionName).findOneAndUpdate((filter || {}), data, options, (err, result) => {
            if (storage === Storage.DBINFO && mongoClient) mongoClient.close();
            if (err) {
              log.warn(`${origin}: Failed to modified data in database ${err}`);
              return callback(`${origin}: Failed modified data in database ${err}`, null);
            }
            log.debug(`${origin}: Data modified in database`);
            return callback(null, result);
          });
        }
      });
    } catch (ex) {
      log.warn(`${origin}: Caught Exception - ${ex}`);
      return callback(`${origin}: Caught Exception - ${ex}`, null);
    }
  }
}

module.exports = DBUtil;
