UNPKG

10.7 kBHTMLView Raw
1<!DOCTYPE html>
2<html lang="en">
3<head>
4 <meta charset="utf-8">
5 <title>JSDoc: Source: loki-fs-structured-adapter.js</title>
6
7 <script src="scripts/prettify/prettify.js"> </script>
8 <script src="scripts/prettify/lang-css.js"> </script>
9 <!--[if lt IE 9]>
10 <script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
11 <![endif]-->
12 <link type="text/css" rel="stylesheet" href="styles/prettify-tomorrow.css">
13 <link type="text/css" rel="stylesheet" href="styles/jsdoc-default.css">
14</head>
15
16<body>
17
18<div id="main">
19
20 <h1 class="page-title">Source: loki-fs-structured-adapter.js</h1>
21
22
23
24
25
26
27 <section>
28 <article>
29 <pre class="prettyprint source linenums"><code>
30/*
31 Loki (node) fs structured Adapter (need to require this script to instance and use it).
32
33 This adapter will save database container and each collection to separate files and
34 save collection only if it is dirty. It is also designed to use a destructured serialization
35 method intended to lower the memory overhead of json serialization.
36
37 This adapter utilizes ES6 generator/iterator functionality to stream output and
38 uses node linereader module to stream input. This should lower memory pressure
39 in addition to individual object serializations rather than loki's default deep object
40 serialization.
41*/
42
43(function (root, factory) {
44 if (typeof define === 'function' &amp;&amp; define.amd) {
45 // AMD
46 define([], factory);
47 } else if (typeof exports === 'object') {
48 // Node, CommonJS-like
49 module.exports = factory();
50 } else {
51 // Browser globals (root is window)
52 root.LokiFsStructuredAdapter = factory();
53 }
54}(this, function () {
55 return (function() {
56
57 const fs = require('fs');
58 const readline = require('readline');
59 const stream = require('stream');
60
61 /**
62 * Loki structured (node) filesystem adapter class.
63 * This class fulfills the loki 'reference' abstract adapter interface which can be applied to other storage methods.
64 *
65 * @constructor LokiFsStructuredAdapter
66 *
67 */
68 function LokiFsStructuredAdapter()
69 {
70 this.mode = "reference";
71 this.dbref = null;
72 this.dirtyPartitions = [];
73 }
74
75 /**
76 * Generator for constructing lines for file streaming output of db container or collection.
77 *
78 * @param {object=} options - output format options for use externally to loki
79 * @param {int=} options.partition - can be used to only output an individual collection or db (-1)
80 *
81 * @returns {string|array} A custom, restructured aggregation of independent serializations.
82 * @memberof LokiFsStructuredAdapter
83 */
84 LokiFsStructuredAdapter.prototype.generateDestructured = function*(options) {
85 var idx, sidx;
86 var dbcopy;
87
88 options = options || {};
89
90 if (!options.hasOwnProperty("partition")) {
91 options.partition = -1;
92 }
93
94 // if partition is -1 we will return database container with no data
95 if (options.partition === -1) {
96 // instantiate lightweight clone and remove its collection data
97 dbcopy = this.dbref.copy();
98
99 for(idx=0; idx &lt; dbcopy.collections.length; idx++) {
100 dbcopy.collections[idx].data = [];
101 }
102
103 yield dbcopy.serialize({
104 serializationMethod: "normal"
105 });
106
107 return;
108 }
109
110 // 'partitioned' along with 'partition' of 0 or greater is a request for single collection serialization
111 if (options.partition >= 0) {
112 var doccount,
113 docidx;
114
115 // dbref collections have all data so work against that
116 doccount = this.dbref.collections[options.partition].data.length;
117
118 for(docidx=0; docidx&lt;doccount; docidx++) {
119 yield JSON.stringify(this.dbref.collections[options.partition].data[docidx]);
120 }
121 }
122 };
123
124 /**
125 * Loki persistence adapter interface function which outputs un-prototype db object reference to load from.
126 *
127 * @param {string} dbname - the name of the database to retrieve.
128 * @param {function} callback - callback should accept string param containing db object reference.
129 * @memberof LokiFsStructuredAdapter
130 */
131 LokiFsStructuredAdapter.prototype.loadDatabase = function(dbname, callback)
132 {
133 var instream = fs.createReadStream(dbname);
134 var outstream = new stream();
135 var rl = readline.createInterface(instream, outstream);
136 var self=this;
137
138 this.dbref = null;
139
140 // first, load db container component
141 rl.on('line', function(line) {
142 // it should single JSON object (a one line file)
143 if (self.dbref === null &amp;&amp; line !== "") {
144 self.dbref = JSON.parse(line);
145 }
146 });
147
148 // when that is done, examine its collection array to sequence loading each
149 rl.on('close', function() {
150 if (self.dbref.collections.length > 0) {
151 self.loadNextCollection(dbname, 0, function() {
152 callback(self.dbref);
153 });
154 }
155 });
156 };
157
158 /**
159 * Recursive function to chain loading of each collection one at a time.
160 * If at some point i can determine how to make async driven generator, this may be converted to generator.
161 *
162 * @param {string} dbname - the name to give the serialized database within the catalog.
163 * @param {int} collectionIndex - the ordinal position of the collection to load.
164 * @param {function} callback - callback to pass to next invocation or to call when done
165 * @memberof LokiFsStructuredAdapter
166 */
167 LokiFsStructuredAdapter.prototype.loadNextCollection = function(dbname, collectionIndex, callback) {
168 var instream = fs.createReadStream(dbname + "." + collectionIndex);
169 var outstream = new stream();
170 var rl = readline.createInterface(instream, outstream);
171 var self=this,
172 obj;
173
174 rl.on('line', function (line) {
175 if (line !== "") {
176 obj = JSON.parse(line);
177 self.dbref.collections[collectionIndex].data.push(obj);
178 }
179 });
180
181 rl.on('close', function (line) {
182 instream = null;
183 outstream = null;
184 rl = null;
185 obj = null;
186
187 // if there are more collections, load the next one
188 if (++collectionIndex &lt; self.dbref.collections.length) {
189 self.loadNextCollection(dbname, collectionIndex, callback);
190 }
191 // otherwise we are done, callback to loadDatabase so it can return the new db object representation.
192 else {
193 callback();
194 }
195 });
196 };
197
198 /**
199 * Generator for yielding sequence of dirty partition indices to iterate.
200 *
201 * @memberof LokiFsStructuredAdapter
202 */
203 LokiFsStructuredAdapter.prototype.getPartition = function*() {
204 var idx,
205 clen = this.dbref.collections.length;
206
207 // since database container (partition -1) doesn't have dirty flag at db level, always save
208 yield -1;
209
210 // yield list of dirty partitions for iterateration
211 for(idx=0; idx&lt;clen; idx++) {
212 if (this.dbref.collections[idx].dirty) {
213 yield idx;
214 }
215 }
216 };
217
218 /**
219 * Loki reference adapter interface function. Saves structured json via loki database object reference.
220 *
221 * @param {string} dbname - the name to give the serialized database within the catalog.
222 * @param {object} dbref - the loki database object reference to save.
223 * @param {function} callback - callback passed obj.success with true or false
224 * @memberof LokiFsStructuredAdapter
225 */
226 LokiFsStructuredAdapter.prototype.exportDatabase = function(dbname, dbref, callback)
227 {
228 var idx;
229
230 this.dbref = dbref;
231
232 // create (dirty) partition generator/iterator
233 var pi = this.getPartition();
234
235 this.saveNextPartition(dbname, pi, function() {
236 callback(null);
237 });
238
239 };
240
241 /**
242 * Utility method for queueing one save at a time
243 */
244 LokiFsStructuredAdapter.prototype.saveNextPartition = function(dbname, pi, callback) {
245 var li;
246 var filename;
247 var self = this;
248 var pinext = pi.next();
249
250 if (pinext.done) {
251 callback();
252 return;
253 }
254
255 // db container (partition -1) uses just dbname for filename,
256 // otherwise append collection array index to filename
257 filename = dbname + ((pinext.value === -1)?"":("." + pinext.value));
258
259 var wstream = fs.createWriteStream(filename);
260 //wstream.on('finish', function() {
261 wstream.on('close', function() {
262 self.saveNextPartition(dbname, pi, callback);
263 });
264
265 li = this.generateDestructured({ partition: pinext.value });
266
267 // iterate each of the lines generated by generateDestructured()
268 for(var outline of li) {
269 wstream.write(outline + "\n");
270 }
271
272 wstream.end();
273 };
274
275 return LokiFsStructuredAdapter;
276
277 }());
278}));
279</code></pre>
280 </article>
281 </section>
282
283
284
285
286</div>
287
288<nav>
289 <h2><a href="index.html">Home</a></h2><h3>Classes</h3><ul><li><a href="Collection.html">Collection</a></li><li><a href="DynamicView.html">DynamicView</a></li><li><a href="Loki.html">Loki</a></li><li><a href="LokiEventEmitter.html">LokiEventEmitter</a></li><li><a href="LokiFsAdapter.html">LokiFsAdapter</a></li><li><a href="LokiFsStructuredAdapter.html">LokiFsStructuredAdapter</a></li><li><a href="LokiIndexedAdapter.html">LokiIndexedAdapter</a></li><li><a href="LokiLocalStorageAdapter.html">LokiLocalStorageAdapter</a></li><li><a href="LokiMemoryAdapter.html">LokiMemoryAdapter</a></li><li><a href="LokiPartitioningAdapter.html">LokiPartitioningAdapter</a></li><li><a href="Resultset.html">Resultset</a></li></ul><h3>Tutorials</h3><ul><li><a href="tutorial-Autoupdating Collections.html">Autoupdating Collections</a></li><li><a href="tutorial-Changes API.html">Changes API</a></li><li><a href="tutorial-Collection Transforms.html">Collection Transforms</a></li><li><a href="tutorial-Indexing and Query performance.html">Indexing and Query performance</a></li><li><a href="tutorial-Loki Angular.html">Loki Angular</a></li><li><a href="tutorial-Persistence Adapters.html">Persistence Adapters</a></li><li><a href="tutorial-Query Examples.html">Query Examples</a></li></ul>
290</nav>
291
292<br class="clear">
293
294<footer>
295 Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 3.4.0</a> on Sun Dec 18 2016 19:39:51 GMT-0500 (Eastern Standard Time)
296</footer>
297
298<script> prettyPrint(); </script>
299<script src="scripts/linenumber.js"> </script>
300</body>
301</html>