Test Report

0
0
36
680
ID Title Duration (ms)
1 matching objects makes a matcher 10
2 callback-friendly JSON.parse doesn't throw an exception 1
3 can freeze can freeze an object to a file 56
4 can freeze can freeze an object to a file 7
5 a large test, >1024 objects can find first object in first bag 3
6 a large test, >1024 objects can find an object twice 4
7 a large test, >1024 objects errors on missing object 3
8 a large test, >1024 objects can find last object in first bag 3
9 a large test, >1024 objects can find first object in second bag 3
10 a large test, >1024 objects can find first object in second bag 4
11 a large test, >1024 objects can find first object in last bag 3
12 a large test, >1024 objects can search for indexed term 6
13 a large test, >1024 objects can search for missing term 37
14 a large test, >1024 objects can find last object in last bag 2
15 a large test, >1024 objects can find range across bags 6
16 a large test, >1024 objects can find range with invalid endpoints (across bags) 4
17 a large test, >1024 objects can find range with invalid endpoints (outside total range) 311
18 an exact multiple test, 1024 objects can find first object in only bag 2
19 an exact multiple test, 1024 objects can find last object in only bag 3
20 an exact multiple test, 1024 objects can search bag 2
21 an exact multiple test, 1024 objects can search for bad term 1
22 an exact multiple test, 1024 objects can find mid range 17
23 zipped pft file can find small range 5
24 create objects creates an object with new 1
25 create objects creates an object without new 0
26 can add and find objects stores an object once added 1
27 can add and find objects handles missing fts 0
28 can add and find objects handles present fts 0
29 can add and find objects handles missing fts & desc 1
30 can add and find objects throws when adding a bad object 1
31 can add and find objects errors when obj not found 0
32 zipped pft file can be thawed 7
33 invalid file reports error on thaw 1
34 thaw errors errors on invalid version 0
35 thaw errors errors on nonexistent file 1
36 thaw errors errors on invalid format 0

Code Coverage Report

98.63%
512
505
7

index.js

100%
2
2
0
Line Hits Source
1 1 'use strict';
2
3 1 module.exports = require('./lib/pure-fts.js');
4

lib/add.js

100%
46
46
0
Line Hits Source
1 1 'use strict';
2
3 1 var stopWords = {
4 'constructor': true,
5 'toString': true,
6 'hasOwnProperty': true,
7 '__proto__': true,
8 'valueOf': true,
9 'npm': true
10 };
11
12 1 function is_stopword(word) {
13 30147 if (word.length <= 3) {
14 12061 return true;
15 }
16
17 18086 return stopWords[word];
18 }
19
20 1 function is_not_stopword(word) {
21 18086 return !is_stopword(word);
22 }
23
24 1 function addfts(purefts, term, name) {
25 12061 if (is_stopword(term)) {
26 6 return;
27 }
28
29 12055 purefts.fts[term] = purefts.fts[term] || {
30 name: term,
31 hits: []
32 };
33
34 12055 purefts.fts[term].hits.push(name);
35 }
36
37 1 function add(obj) {
38 6032 if (!obj || !obj.name) {
39 2 throw new Error("Cannot add object %j: no `name` member", obj);
40 }
41
42 6030 var purefts = this;
43
44 6030 purefts.keys.push(obj.name);
45
46 // add fts entries
47 6030 if (!obj.fts) {
48 3 obj.fts = obj.description || "";
49 }
50
51 6030 obj.fts.split(' ').filter(is_not_stopword)
52 .forEach(function (term) {
53 6031 addfts(purefts, term, obj.name);
54 });
55 6030 addfts(purefts, obj.name, obj.name);
56
57 6030 purefts.values[obj.name] = obj;
58 }
59
60 1 module.exports = add;
61

lib/clean.js

100%
7
7
0
Line Hits Source
1 1 'use strict';
2
3 1 function clean() {
4 4 var purefts = this;
5
6 4 purefts.keys = purefts.keys.sort();
7
8 4 return;
9 }
10
11 1 module.exports = clean;
12

lib/findIndex.js

92.31%
26
24
2
Line Hits Source
1 1 'use strict';
2
3 1 var bsearch = require('binary-search');
4
5 1 function findIndex(config, name, cb) {
6 98 var bag_idx,
7 bag,
8 idx;
9
10 98 bag_idx = bsearch(config.index, name, function (a, b) {
11 513 return a.last < b ? -1 : (a.first > b ? 1 : 0);
12 });
13
14 98 if (bag_idx < 0) {
15 2 return cb(null);
16 }
17
18 96 bag = config.index[bag_idx];
19
20 96 config.getFile(bag.name, function (err, bagValues) {
21
if (
err
) {
22 return cb(err);
23 }
24
25 96 idx = bsearch(bagValues, name, function (a, b) {
26 572 var n = config.getName(a);
27 572 return n < b ? -1 : n > b;
28 });
29
30 96 return cb(null, (idx < 0) ? undefined : bagValues[idx],
31 bagValues);
32
33 });
34 }
35
36 1 module.exports = findIndex;
37

lib/findRange.js

95.56%
45
43
2
Line Hits Source
1 1 'use strict';
2
3 1 var bsearch = require('binary-search');
4 1 var async = require('async');
5
6 1 function findBag(config, term) {
7 10 return bsearch(config.index, term, function (a, b) {
8 11 return a.last < b ? -1 : (a.first > b ? 1 : 0);
9 });
10 }
11
12 1 function findRange(start, end, cb, done) {
13 5 var p = this,
14 config = p.keyConfig,
15 start_bag_idx,
16 end_bag_idx,
17 i,
18 bags = [];
19
20 // find first bag
21 5 start_bag_idx = findBag(config, start);
22
23 5 if (start_bag_idx < 0) {
24 1 start_bag_idx = 0;
25 }
26
27 // find ending bag
28 5 end_bag_idx = findBag(config, end);
29
30 5 if (end_bag_idx < 0) {
31 1 end_bag_idx = config.index.length - 1;
32 }
33
34 5 for (i = start_bag_idx; i <= end_bag_idx; i += 1) {
35 6 bags.push(i);
36 }
37
38 5 async.eachSeries(bags, function (bag_idx, callback) {
39 6 var bag = config.index[bag_idx];
40
41 6 config.getFile(bag.name, function (err, bagValues) {
42
if (
err
) {
43 return cb(err);
44 }
45
46 6 bagValues.filter(function (n) {
47 15240 return (start <= n && n <= end);
48 }).forEach(function (n) {
49 5010 cb(null, n);
50 });
51 6 callback();
52 });
53 }, done);
54
55 5 return;
56 }
57
58 1 module.exports = findRange;
59

lib/freeze.js

100%
38
38
0
Line Hits Source
1 1 'use strict';
2
3 1 var fs = require('fs');
4 1 var mkdirp = require('mkdirp');
5 1 var rimraf = require('rimraf');
6 1 var Zip = require('adm-zip');
7 1 var async = require('async');
8
9 1 var fts = require('./fts');
10 1 var makeIndex = require('./makeIndex');
11 1 var indexConfig = require('./indexConfig');
12
13 1 var version = {
14 file: "4.0.0"
15 };
16
17 1 function freeze(file, cb) {
18 4 var p = this;
19 4 p.clean();
20
21 4 p.putFile = function (filename, value, cb) {
22 132 fs.writeFile(file + filename, JSON.stringify(value), cb);
23 };
24
25 4 rimraf(file, function () {
26
27 4 mkdirp.sync(file + "/data/");
28
29 4 var tasks = [
30 function (callback) {
31 4 p.putFile("/data/version.json", version, callback);
32 },
33 function (callback) {
34 4 makeIndex(indexConfig.key, p.keys, p.putFile, callback);
35 },
36 function (callback) {
37 4 makeIndex(indexConfig.val, p.values, p.putFile, callback);
38 },
39 function (callback) {
40 4 makeIndex(indexConfig.fts, fts(p), p.putFile, callback);
41 }
42 ];
43
44 4 async.series(tasks, cb);
45 });
46 }
47
48 1 module.exports = freeze;
49

lib/fts.js

100%
19
19
0
Line Hits Source
1 1 'use strict';
2
3 1 function percentLimit(value, percent) {
4 4 return Math.max(10, Math.ceil(percent * value));
5 }
6
7 1 function prepFts(p) {
8 4 var newFts = {},
9 limit,
10 ftsKeys;
11
12 // first remove that are too frequent hits
13 4 limit = percentLimit(p.keys.length, 0.02);
14
15 4 ftsKeys = Object.keys(p.fts).filter(function (term) {
16 6028 return p.fts[term].hits.length <= limit;
17 6026 }).filter(function (key) { return key; });
18
19 4 ftsKeys.forEach(function (term) {
20 6026 newFts[term] = p.fts[term];
21 });
22
23 4 return newFts;
24 }
25
26 1 module.exports = prepFts;
27
28

lib/get.js

100%
26
26
0
Line Hits Source
1 1 'use strict';
2
3 1 var util = require('util');
4 1 var bsearch = require('binary-search');
5
6 1 var findIndex = require('./findIndex');
7
8 1 function get(name, cb) {
9 21 var p = this,
10 v;
11
12 21 setImmediate(function () {
13 21 if (!name) {
14 1 return cb(new Error(util.format('Cannot find object with invalid name %j',
15 name)));
16 }
17
18 20 v = p.values[name];
19 20 if (v) {
20 4 return cb(null, v);
21 }
22
23 16 findIndex(p.valConfig, name, function (err, value) {
24
25 16 p.values[name] = value;
26
27 16 if (!value) {
28 2 err = new Error(util.format('No object with name %j', name));
29 }
30
31 16 return cb(err, value);
32 });
33
34 });
35 }
36
37 1 module.exports = get;
38

lib/indexConfig.js

100%
25
25
0
Line Hits Source
1 1 'use strict';
2
3 1 function identity(x) {
4 10 return x;
5 }
6
7 1 function dotName(x) {
8 794 return x.name;
9 }
10
11 1 var indexConfig = {
12 key: {
13 getName: identity,
14 prefix: "/data/key",
15 blockSize: 4096
16 },
17 fts: {
18 getName: dotName,
19 prefix: "/data/fts",
20 blockSize: 512
21 },
22 val: {
23 getName: dotName,
24 prefix: "/data/val",
25 blockSize: 64
26 }
27 };
28
29 1 module.exports = indexConfig;
30

lib/loadIndexes.js

100%
26
26
0
Line Hits Source
1 1 'use strict';
2
3 1 var indexConfig = require('./indexConfig');
4 1 var async = require('async');
5
6 1 function makeTask(p, n, indexConfig) {
7 66 var cfg = indexConfig[n];
8
9 66 cfg.getFile = p.getFile;
10 66 p[n + 'Config'] = cfg;
11
12 66 return function (callback) {
13 66 var file = cfg.prefix + '_index.json';
14 66 p.getFile(file, function (err, data) {
15 66 cfg.index = data;
16 66 callback(err);
17 });
18 };
19 }
20
21
22 1 function loadIndexes(p, cb) {
23 22 var tasks = [];
24
25 22 Object.keys(indexConfig).forEach(function (n) {
26 66 var task = makeTask(p, n, indexConfig);
27
28 66 tasks.push(task);
29 });
30
31 22 async.series(tasks, function () {
32 22 cb(null, p);
33 });
34 }
35
36
37 1 module.exports = loadIndexes;
38

lib/makeIndex.js

98.46%
65
64
1
Line Hits Source
1 1 'use strict';
2
3 1 function zeroFill(d) {
4 116 var s = d.toString(36);
5 116 while (s.length < 4) {
6 305 s = '0' + s;
7 }
8 116 return s;
9 }
10
11 1 function makeIndex(config, object, putFile, cb) {
12 12 var i = 0,
13 bag = [],
14 index = [],
15 keys,
16 getValue,
17 ended = false,
18 inFlight = 0,
19 bagCount = 0;
20
21 12 if (Array.isArray(object)) {
22 4 keys = [].concat(object);
23 4 keys.sort(config.compareNames);
24 6030 getValue = function (key) { return key; };
25 } else {
26 8 keys = Object.keys(object).sort(config.compareNames);
27 12060 getValue = function (key) { return object[key]; };
28 }
29
30 12 function decrement() {
31 128 inFlight -= 1;
32
if (
ended
&& inFlight === 0) {
33 12 cb(null);
34 }
35 }
36
37 12 function localPutFile(name, value) {
38 128 inFlight += 1;
39 128 putFile(name, value, decrement);
40 }
41
42 12 function makeIndexEntry(bag, bagCount) {
43 116 return {
44 index: bagCount,
45 name: config.prefix + zeroFill(bagCount) + ".json",
46 first: config.getName(bag[0]),
47 last: config.getName(bag[bag.length - 1])
48 };
49 }
50
51 12 function pushIndexEntry() {
52 116 var indexEntry = makeIndexEntry(bag, bagCount);
53
54 // write this bag
55 116 localPutFile(indexEntry.name, bag);
56
57 116 index.push(indexEntry);
58
59 116 bag = [];
60 116 bagCount += 1;
61 }
62
63 12 for (i = 0; i < keys.length; i += 1) {
64
65 18078 bag.push(getValue(keys[i]));
66
67 18078 if (bag.length === config.blockSize) {
68 106 pushIndexEntry();
69 }
70 }
71
72 // last bag
73 12 if (bag.length) {
74 10 pushIndexEntry();
75 }
76
77 12 localPutFile(config.prefix + "_index.json", index);
78 12 ended = true;
79 }
80
81 1 module.exports = makeIndex;
82

lib/pure-fts.js

100%
18
18
0
Line Hits Source
1 1 'use strict';
2
3 1 function Purefts() {
4 35 var p = this;
5 35 if (!(p instanceof Purefts)) {
6 1 return new Purefts(arguments);
7 }
8
9 34 p.add = require('./add');
10 34 p.clean = require('./clean');
11
12 34 p.get = require('./get');
13 34 p.search = require('./search');
14
15 34 p.findRange = require('./findRange');
16
17 34 p.freeze = require('./freeze');
18
19 34 p.keys = [];
20 34 p.values = {};
21 34 p.fts = {};
22 }
23
24 1 module.exports = Purefts;
25
26 1 Purefts.thaw = require('./thaw');
27

lib/search.js

100%
56
56
0
Line Hits Source
1 1 'use strict';
2
3 1 var async = require('async');
4 1 var findIndex = require('./findIndex');
5
6 1 var Hoek = require('Hoek');
7 1 var through = require('through');
8
9 1 function makeMatcher(term) {
10 4 return function (val) {
11 5005 if (val.name === term) {
12 3 return true;
13 }
14
15 5002 if (val.fts.indexOf(term) >= 0) {
16 1 return true;
17 }
18
19 5001 return false;
20 };
21 }
22
23
24 1 function search(term, cb, done) {
25 3 var p = this,
26 match = makeMatcher(term),
27 keys;
28
29 3 findIndex(p.ftsConfig, term, function (err, fts) {
30
31 3 var s = through(function (chunk) {
32 5002 if (match(chunk)) {
33 2 cb(err, chunk);
34 }
35 }, done);
36
37 3 if (fts) {
38 2 keys = Hoek.unique(fts.hits);
39
40 // if we group keys into bags
41 // can avoid re-reading a bag multiple times
42
43 2 async.eachLimit(keys, 8, function (k, cont) {
44 2 p.get(k, function (err, val) {
45 2 s.write(val);
46 2 cont(err);
47 });
48 }, function () {
49 2 s.end();
50 });
51 2 return;
52 }
53
54 // each value
55 1 async.eachLimit(p.valConfig.index, 8, function (k, cont) {
56 79 findIndex(p.valConfig, k.first, function (err, value, bag) {
57 /*jslint unparam: true*/
58 79 bag.forEach(function (val) {
59 5000 s.write(val);
60 });
61
62 79 cont(err);
63 });
64 }, function () {
65 1 s.end();
66 });
67
68 });
69 }
70
71 1 module.exports = search;
72 1 search.makeMatcher = makeMatcher;
73

lib/thaw.js

98.23%
113
111
2
Line Hits Source
1 1 'use strict';
2
3 1 var fs = require('fs');
4 1 var Zip = require('adm-zip');
5 1 var util = require('util');
6 1 var async = require('async');
7 1 var JSONStream = require('JSONStream');
8
9 1 var Purefts = require('./pure-fts');
10
11 1 var loadIndexes = require('./loadIndexes');
12
13 // supported data versions : only one right now
14 1 function thaw_4_0_0(getFile, cb) {
15 22 var p = new Purefts();
16
17 22 delete p.keys;
18 22 p.fts = {};
19
20 /* give p an appropriate async file getter */
21 22 p.getFile = getFile;
22
23 22 loadIndexes(p, cb);
24 }
25
26 1 function invalidVersion(version) {
27 2 return function (z, cb) {
28 /*jslint unparam:true*/
29 2 return cb(new Error(util.format("unknown version %j", version)));
30 };
31 }
32
33 1 var versions = {
34 "4.0.0": thaw_4_0_0
35 };
36
37 1 function chooseVersion(version) {
38 24 var thawer = versions[version];
39 24 if (!thawer) {
40 2 thawer = invalidVersion(version);
41 }
42 24 return thawer;
43 }
44
45 // convert thrown exceptions into callback err
46 1 function parseJSON(buf, cb) {
47 192 try {
48 192 cb(null, JSON.parse(buf));
49 } catch (err) {
50 1 return cb(err);
51 }
52 }
53
54 // supported storage formats: orthogonal to version
55 1 function makeDirGetFile(file) {
56 25 return function (filename, cb) {
57 181 fs.readFile(file + filename, function (err, buf) {
58 181 if (err) {
59 5 return cb(err);
60 }
61
62 176 parseJSON(buf, cb);
63 });
64 };
65 }
66
67 1 function makeZipGetFile(file) {
68 25 var zip = new Zip(file);
69
70 3 return function (name, cb) {
71 15 zip.readAsTextAsync(name.substring(1), function (data, err) {
72
if (
err
) {
73 return cb(err);
74 }
75
76 15 parseJSON(data, cb);
77 });
78 };
79 }
80
81 1 function makeThawFormatReader(makeGetFile) {
82 2 return function (file, cb) {
83 50 var getFile,
84 thawer;
85
86 50 try {
87 50 getFile = makeGetFile(file);
88
89 28 getFile('/data/version.json', function (err, version) {
90 28 if (err) {
91 5 return cb(err);
92 }
93
94 23 thawer = chooseVersion(version.file);
95
96 23 thawer(getFile, cb);
97 });
98 } catch (err) {
99 22 return cb(err);
100 }
101 };
102 }
103
104 1 var thawFormats = [
105 makeThawFormatReader(makeDirGetFile),
106 makeThawFormatReader(makeZipGetFile)
107 ];
108
109
110
111 1 function thaw(file, cb) {
112 25 var errs = [];
113
114 // try each known format
115 25 async.each(thawFormats, function (t, next) {
116 50 t(file, function (err, p) {
117
118 // if failed, store error
119 50 if (err) {
120 28 errs.push(String(err));
121
122 // and continue to next format
123 28 return next(null);
124 }
125
126 // thawed it, so terminate the async each
127 22 next(true);
128
129 22 cb(null, p);
130 });
131 }, function (found) {
132 25 if (found) {
133 22 return;
134 }
135
136 // report all errors from each attempted format
137 3 cb(new Error("Could not thaw " + file + ": unknown format\n ",
138 errs.join("\n ")));
139 });
140 }
141
142 1 module.exports = thaw;
143
144 /* export for testing */
145 1 thaw.chooseThawer = chooseVersion;
146 1 thaw.parseJSON = parseJSON;
147