JSONStream/test/gen.js

return // dont run this test for now since tape is weird and broken on 0.10

var fs = require('fs')
var JSONStream = require('../')
var file = process.argv[2] || '/tmp/JSONStream-test-large.json'
var size = Number(process.argv[3] || 100000)
var tape = require('tape')
// if (process.title !== 'browser') {
  tape('out of mem', function (t) {
    t.plan(1)
    //////////////////////////////////////////////////////
    // Produces a random number between arg1 and arg2
    //////////////////////////////////////////////////////
    var randomNumber = function (min, max) {
      var number = Math.floor(Math.random() * (max - min + 1) + min);
      return number;
    };

    //////////////////////////////////////////////////////
    // Produces a random string of a length between arg1 and arg2
    //////////////////////////////////////////////////////
    var randomString = function (min, max) {

      // add several spaces to increase chanses of creating 'words'
      var chars = '      0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
      var result = '';

      var randomLength = randomNumber(min, max);

      for (var i = randomLength; i > 0; --i) {
        result += chars[Math.round(Math.random() * (chars.length - 1))];
      }
      return result;
    };

    //////////////////////////////////////////////////////
    // Produces a random JSON document, as a string
    //////////////////////////////////////////////////////
    var randomJsonDoc = function () {

      var doc = {
        "CrashOccurenceID": randomNumber(10000, 50000),
        "CrashID": randomNumber(1000, 10000),
        "SiteName": randomString(10, 25),
        "MachineName": randomString(10, 25),
        "Date": randomString(26, 26),
        "ProcessDuration": randomString(18, 18),
        "ThreadIdentityName": null,
        "WindowsIdentityName": randomString(15, 40),
        "OperatingSystemName": randomString(35, 65),
        "DetailedExceptionInformation": randomString(100, 800)
      };

      doc = JSON.stringify(doc);
      doc = doc.replace(/\,/g, ',\n'); // add new lines after each attribute
      return doc;
    };

    //////////////////////////////////////////////////////
    // generates test data
    //////////////////////////////////////////////////////
    var generateTestData = function (cb) {

      console.log('generating large data file...');

      var stream = fs.createWriteStream(file, {
        encoding: 'utf8'
      });

      var i = 0;
      var max = size;
      var writing = false
      var split = ',\n';
      var doc = randomJsonDoc();
      stream.write('[');

      function write () {
        if(writing) return
        writing = true
        while(++i < max) {
          if(Math.random() < 0.001)
            console.log('generate..', i + ' / ' + size)
          if(!stream.write(doc + split)) {
            writing = false
            return stream.once('drain', write)
          }
        }
        stream.write(doc + ']')
        stream.end();
        console.log('END')
      }
      write()
      stream.on('close', cb)
    };

    //////////////////////////////////////////////////////
    // Shows that parsing 100000 instances using JSONStream fails
    //
    // After several seconds, you will get this crash
    //     FATAL ERROR: JS Allocation failed - process out of memory
    //////////////////////////////////////////////////////
    var testJSONStreamParse_causesOutOfMem = function (done) {
      var items = 0
      console.log('parsing data files using JSONStream...');

      var parser = JSONStream.parse([true]);
      var stream = fs.createReadStream(file);
      stream.pipe(parser);

      parser.on('data', function (data) {
        items++
        if(Math.random() < 0.01) console.log(items, '...')
      });
      
      parser.on('end', function () {
        t.equal(items, size)
      });

    };

    //////////////////////////////////////////////////////
    // main
    //////////////////////////////////////////////////////

    fs.stat(file, function (err, stat) {
      console.log(stat)
      if(err)
        generateTestData(testJSONStreamParse_causesOutOfMem);
      else 
        testJSONStreamParse_causesOutOfMem()
    })

  })
  
// }