1 | <!DOCTYPE html>
|
2 | <html lang="en">
|
3 | <head>
|
4 |
|
5 | <meta charset="utf-8">
|
6 | <title>lib/parse_stream.js - Documentation</title>
|
7 |
|
8 | <meta name="description" content="Implementation of the v2 twitter streaming api" />
|
9 |
|
10 | <meta name="keywords" content="twitter api v2 node.js rest http stream client" />
|
11 | <meta name="keyword" content="twitter api v2 node.js rest http stream client" />
|
12 |
|
13 |
|
14 |
|
15 | <script src="scripts/prettify/prettify.js"></script>
|
16 | <script src="scripts/prettify/lang-css.js"></script>
|
17 | |
18 |
|
19 |
|
20 | <link type="text/css" rel="stylesheet" href="styles/prettify.css">
|
21 | <link type="text/css" rel="stylesheet" href="styles/jsdoc.css">
|
22 | <script src="scripts/nav.js" defer></script>
|
23 | <meta name="viewport" content="width=device-width, initial-scale=1.0">
|
24 | </head>
|
25 | <body>
|
26 |
|
27 | <input type="checkbox" id="nav-trigger" class="nav-trigger" />
|
28 | <label for="nav-trigger" class="navicon-button x">
|
29 | <div class="navicon"></div>
|
30 | </label>
|
31 |
|
32 | <label for="nav-trigger" class="overlay"></label>
|
33 |
|
34 | <nav >
|
35 |
|
36 | <input type="text" id="nav-search" placeholder="Search" />
|
37 |
|
38 | <h2><a href="index.html">Home</a></h2><h2><a href="https://github.com/CrunchwrapSupreme/twit-stream-v2" target="_blank" class="menu-item" id="website_link" >Github Repo</a></h2><h3>Classes</h3><ul><li><a href="StreamClient.html">StreamClient</a><ul class='methods'><li data-type='method'><a href="StreamClient.html#connect">connect</a></li><li data-type='method'><a href="StreamClient.html#disconnect">disconnect</a></li></ul></li></ul><h3>Events</h3><ul><li><a href="StreamClient.html#event:tweet">tweet</a></li><li><a href="StreamClient.html#event:heartbeat">heartbeat</a></li><li><a href="StreamClient.html#event:stream-error">stream-error</a></li><li><a href="StreamClient.html#event:api-errors">api-errors</a></li><li><a href="StreamClient.html#event:other">other</a></li><li><a href="StreamClient.html#event:connected">connected</a></li><li><a href="StreamClient.html#event:reconnect">reconnect</a></li><li><a href="StreamClient.html#event:disconnected">disconnected</a></li><li><a href="StreamClient.html#event:close">close</a></li></ul>
|
39 | </nav>
|
40 |
|
41 | <div id="main">
|
42 |
|
43 | <h1 class="page-title">lib/parse_stream.js</h1>
|
44 |
|
45 |
|
46 |
|
47 |
|
48 |
|
49 |
|
50 |
|
51 | <section>
|
52 | <article>
|
53 | <pre class="prettyprint source linenums"><code>const { Transform } = require('stream');
|
54 | const MAX_BUFFER_SIZE = 1024 * 10;
|
55 |
|
56 | class TweetStreamParser extends Transform {
|
57 | constructor(options) {
|
58 | super({
|
59 | ...options,
|
60 | readableObjectMode: true
|
61 | });
|
62 |
|
63 | if(options) {
|
64 | this.emitr = options.emitter;
|
65 | this.timer = options.timer;
|
66 | }
|
67 | this.emitr = this.emitr || this;
|
68 | }
|
69 |
|
70 | parseJSON() {
|
71 | const EOF = '\r\n';
|
72 | const emitter = this.emitr;
|
73 | let error, index;
|
74 | while( (index = this.chunkBuffer.indexOf(EOF, 'utf8')) > -1 ) {
|
75 | let chunk = this.chunkBuffer.toString('utf8', 0, index);
|
76 | this.chunkBuffer = this.chunkBuffer.slice(index + EOF.length);
|
77 | if(chunk.length > 0) {
|
78 | try {
|
79 | let json = JSON.parse(chunk);
|
80 | if(json.data) {
|
81 | emitter.emit('tweet', json);
|
82 | this.push(json);
|
83 | } else if(json.errors) {
|
84 | emitter.emit('api-errors', json);
|
85 | } else {
|
86 | emitter.emit('other', json);
|
87 | }
|
88 | } catch(json_error) {
|
89 | json_error.source = chunk;
|
90 | error = json_error;
|
91 | this.emit('stream-error', error);
|
92 | }
|
93 | } else {
|
94 | emitter.emit('heartbeat');
|
95 | }
|
96 | }
|
97 |
|
98 | //return error;
|
99 | }
|
100 |
|
101 | _transform(chunk, encoding, callback) {
|
102 | this.encoding = encoding;
|
103 | let op_error;
|
104 | if(this.chunkBuffer && this.chunkBuffer.length + chunk.length < MAX_BUFFER_SIZE) {
|
105 | const err_info = {
|
106 | lastChunk: chunk,
|
107 | buffer: this.chunkBuffer,
|
108 | max_size_b: MAX_BUFFER_SIZE
|
109 | };
|
110 | op_error = new Error(`Stream overproducing tweet data\n${err_info}`);
|
111 | } else if(this.chunkBuffer && chunk) {
|
112 | this.chunkBuffer = Buffer.concat([this.chunkBuffer, chunk]);
|
113 | } else if(chunk) {
|
114 | this.chunkBuffer = chunk;
|
115 | }
|
116 | let parse_error = this.parseJSON();
|
117 | if(this.timer) {
|
118 | this.timer.refresh();
|
119 | }
|
120 | return callback(op_error || parse_error);
|
121 | }
|
122 |
|
123 | /**
|
124 | * Clear the chunk buffer
|
125 | */
|
126 | _flush(callback) {
|
127 | let error;
|
128 | if(this.chunkBuffer) {
|
129 | error = this.parseJSON();
|
130 | this.chunkBuffer = null;
|
131 | }
|
132 | return callback(error);
|
133 | }
|
134 | }
|
135 |
|
136 | module.exports = TweetStreamParser;
|
137 | </code></pre>
|
138 | </article>
|
139 | </section>
|
140 |
|
141 |
|
142 |
|
143 |
|
144 |
|
145 |
|
146 | </div>
|
147 |
|
148 | <br class="clear">
|
149 |
|
150 | <footer>
|
151 | Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 3.6.6</a> on Mon Oct 19 2020 05:12:10 GMT-0500 (Central Daylight Time) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
|
152 | </footer>
|
153 |
|
154 | <script>prettyPrint();</script>
|
155 | <script src="scripts/polyfill.js"></script>
|
156 | <script src="scripts/linenumber.js"></script>
|
157 |
|
158 | <script src="scripts/search.js" defer></script>
|
159 |
|
160 |
|
161 |
|
162 | </body>
|
163 | </html>
|