UNPKG

5.16 kBHTMLView Raw
1<!DOCTYPE html>
2<html lang="en">
3<head>
4
5 <meta charset="utf-8">
6 <title>lib/parse_stream.js - Documentation</title>
7
8 <meta name="description" content="Implementation of the v2 twitter streaming api" />
9
10 <meta name="keywords" content="twitter api v2 node.js rest http stream client" />
11 <meta name="keyword" content="twitter api v2 node.js rest http stream client" />
12
13
14
15 <script src="scripts/prettify/prettify.js"></script>
16 <script src="scripts/prettify/lang-css.js"></script>
17 <!--[if lt IE 9]>
18 <script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
19 <![endif]-->
20 <link type="text/css" rel="stylesheet" href="styles/prettify.css">
21 <link type="text/css" rel="stylesheet" href="styles/jsdoc.css">
22 <script src="scripts/nav.js" defer></script>
23 <meta name="viewport" content="width=device-width, initial-scale=1.0">
24</head>
25<body>
26
27<input type="checkbox" id="nav-trigger" class="nav-trigger" />
28<label for="nav-trigger" class="navicon-button x">
29 <div class="navicon"></div>
30</label>
31
32<label for="nav-trigger" class="overlay"></label>
33
34<nav >
35
36 <input type="text" id="nav-search" placeholder="Search" />
37
38 <h2><a href="index.html">Home</a></h2><h2><a href="https://github.com/CrunchwrapSupreme/twit-stream-v2" target="_blank" class="menu-item" id="website_link" >Github Repo</a></h2><h3>Classes</h3><ul><li><a href="StreamClient.html">StreamClient</a><ul class='methods'><li data-type='method'><a href="StreamClient.html#connect">connect</a></li><li data-type='method'><a href="StreamClient.html#disconnect">disconnect</a></li></ul></li></ul><h3>Events</h3><ul><li><a href="StreamClient.html#event:tweet">tweet</a></li><li><a href="StreamClient.html#event:heartbeat">heartbeat</a></li><li><a href="StreamClient.html#event:stream-error">stream-error</a></li><li><a href="StreamClient.html#event:api-errors">api-errors</a></li><li><a href="StreamClient.html#event:other">other</a></li><li><a href="StreamClient.html#event:connected">connected</a></li><li><a href="StreamClient.html#event:reconnect">reconnect</a></li><li><a href="StreamClient.html#event:disconnected">disconnected</a></li><li><a href="StreamClient.html#event:close">close</a></li></ul>
39</nav>
40
41<div id="main">
42
43 <h1 class="page-title">lib/parse_stream.js</h1>
44
45
46
47
48
49
50
51 <section>
52 <article>
53 <pre class="prettyprint source linenums"><code>const { Transform } = require('stream');
54const MAX_BUFFER_SIZE = 1024 * 10;
55
56class TweetStreamParser extends Transform {
57 constructor(options) {
58 super({
59 ...options,
60 readableObjectMode: true
61 });
62
63 if(options) {
64 this.emitr = options.emitter;
65 this.timer = options.timer;
66 }
67 this.emitr = this.emitr || this;
68 }
69
70 parseJSON() {
71 const EOF = '\r\n';
72 const emitter = this.emitr;
73 let error, index;
74 while( (index = this.chunkBuffer.indexOf(EOF, 'utf8')) > -1 ) {
75 let chunk = this.chunkBuffer.toString('utf8', 0, index);
76 this.chunkBuffer = this.chunkBuffer.slice(index + EOF.length);
77 if(chunk.length > 0) {
78 try {
79 let json = JSON.parse(chunk);
80 if(json.data) {
81 emitter.emit('tweet', json);
82 this.push(json);
83 } else if(json.errors) {
84 emitter.emit('api-errors', json);
85 } else {
86 emitter.emit('other', json);
87 }
88 } catch(json_error) {
89 json_error.source = chunk;
90 error = json_error;
91 this.emit('stream-error', error);
92 }
93 } else {
94 emitter.emit('heartbeat');
95 }
96 }
97
98 //return error;
99 }
100
101 _transform(chunk, encoding, callback) {
102 this.encoding = encoding;
103 let op_error;
104 if(this.chunkBuffer &amp;&amp; this.chunkBuffer.length + chunk.length &lt; MAX_BUFFER_SIZE) {
105 const err_info = {
106 lastChunk: chunk,
107 buffer: this.chunkBuffer,
108 max_size_b: MAX_BUFFER_SIZE
109 };
110 op_error = new Error(`Stream overproducing tweet data\n${err_info}`);
111 } else if(this.chunkBuffer &amp;&amp; chunk) {
112 this.chunkBuffer = Buffer.concat([this.chunkBuffer, chunk]);
113 } else if(chunk) {
114 this.chunkBuffer = chunk;
115 }
116 let parse_error = this.parseJSON();
117 if(this.timer) {
118 this.timer.refresh();
119 }
120 return callback(op_error || parse_error);
121 }
122
123 /**
124 * Clear the chunk buffer
125 */
126 _flush(callback) {
127 let error;
128 if(this.chunkBuffer) {
129 error = this.parseJSON();
130 this.chunkBuffer = null;
131 }
132 return callback(error);
133 }
134}
135
136module.exports = TweetStreamParser;
137</code></pre>
138 </article>
139 </section>
140
141
142
143
144
145
146</div>
147
148<br class="clear">
149
150<footer>
151 Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 3.6.6</a> on Mon Oct 19 2020 05:12:10 GMT-0500 (Central Daylight Time) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
152</footer>
153
154<script>prettyPrint();</script>
155<script src="scripts/polyfill.js"></script>
156<script src="scripts/linenumber.js"></script>
157
158<script src="scripts/search.js" defer></script>
159
160
161
162</body>
163</html>