| 1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230 | 1
1
1
18
18
18
18
34
1
1
1
1
1
28
28
28
1
16
7
7
7
7
7
1
28
4784
1
4784
4784
748
76
76
76
672
671
1
748
3338
2883
455
103
352
351
351
1
3338
482
380
380
102
1
101
482
104
100
4
3
3
1
1
1
1
104
99
62
62
62
62
37
11
26
26
26
99
11
9
9
9
9
2
1
1
1
1
1
1
11
2
1
| var Stream = require('stream');
var util = require('util');
// Creates a reusable tokenizer which inherits from
// the Stream module.
function HeaderTokenizer() {
// Call the Stream.Transform constructor
Stream.Transform.call(this);
// Initialize state
this.state = this.states.FIELD_NAME;
this.currentToken = {type: 'field_name', value: ""};
// If someone attaches a new event listener for the 'done' event after
// we are already in the DONE state, then just re-fire the event.
this.on('newListener', function(name, listener) {
if (name === 'done' && this.state === this.states.DONE) {
listener();
}
});
}
// Set up the inheritance
util.inherits(HeaderTokenizer, Stream.Transform);
// Helper object to hold the regular expressions for matching various
// character types.
HeaderTokenizer.prototype.charTypes = {
colon: /\:/,
newline: /\n/,
whitespace: /\s/,
carraigeReturn: /\r/,
printableAscii: /[\x21-\x7e]/
};
HeaderTokenizer.prototype.states = {
// While processing a field name the tokenizer is in this state.
FIELD_NAME: 'FIELD_NAME',
// While processing printable characters in the field body the tokenizer
// is in this state.
FIELD_BODY: 'FIELD_BODY',
// While processing whitespace characters in the field body the
// tokenizer is in this state. All consecutive whitespace characters are
// collapsed into a single space.
WHITESPACE: 'WHITESPACE',
// While processing a <CR> in the field body the tokenizer is in this
// state.
CARRAIGE_RETURN: 'CARRAIGE_RETURN',
// While processing stuff after a <CR><LF> the tokenizer is in this
// state. Note that header field bodies are allowed to "fold," meaning
// that they can carry onto multiple lines as long as the "folded over"
// portion is indented with at least one linear white space character.
CRLF: 'CRLF',
// While processing stuff after a <CR><LF><CR> the tokenizer is in this
// state. Significant because the tokenizer expects a <LF> and will emit
// an error if it receives anything else.
CRLF_CR: 'CRLF_CR',
// After consuming an empty line, the tokenizer emits a "done" event and
// transitions to this trap state. TODO: deprecate the "done" event
// since the message parser will probably never even send the header
// tokenier the require CRLFCRLF characters required to get to this done
// state.
DONE: 'DONE'
};
// Implements the Stream.Transform._transform method.
HeaderTokenizer.prototype._transform = function(chunk, encoding, callback) {
var str = chunk.toString('binary');
this.tokenize(str);
callback();
};
HeaderTokenizer.prototype._flush = function(callback) {
// Assert: we are done consuming the readable stream. Emit the current
// token. If we have already entered the DONE state then do not emit
// anything. We do this check because there are two cases when we want
// to emit the 'done' event:
// 1. After encountering <CR><LF><CR><LF> sequence in an email message.
// When this happens, the HeaderTokenizer immediately enters the DONE
// state.
// 2. After encountering the end of a stream in an email message which
// contained no message body (and thus potentially no <CR><LF><CR><LF>
// sequence).
if (this.state !== this.states.DONE) {
this.state = this.states.DONE;
this.currentToken.value = this.currentToken.value.trim();
this.emit('token', this.currentToken);
this.emit('done');
callback();
}
};
// The tokenize function simply takes in a string and feeds each character
// to the HeaderTokenizer#receiveChar function.
HeaderTokenizer.prototype.tokenize = function(str) {
for (var i = 0; i < str.length; i++) {
this.receiveChar(str[i]);
}
};
// This function is the heart of the HeaderTokenizer class. It consumes
// the characters of an email message sequentially, emitting
// "token" events when it has finished tokenizing header field names and
// header field bodies. The "token" events have a
// header token object of the form `{type:'field_body', value:'blah'}` as
// the argument for their listener functions.
HeaderTokenizer.prototype.receiveChar = function(ch) {
var states = this.states; // Cache the states
switch(this.state) {
case this.states.FIELD_NAME:
if (this.charTypes.colon.test(ch)) {
// Colon delimits fieldName:fieldBody. Emit the current
// field name token and set up the new field body token
this.emit('token', this.currentToken);
this.currentToken = {type: 'field_body', value: ""};
this.state = states.FIELD_BODY;
} else if (this.charTypes.printableAscii.test(ch)) {
// Append to the current field name
this.currentToken.value += ch;
} else {
// Whitespace, or other non-printable ascii characters are
// not allowed in the field names
this.emit('error', new Error('Unexpected character "' + ch + '" while parsing field name.'));
}
break;
case this.states.FIELD_BODY:
if (this.charTypes.printableAscii.test(ch)) {
this.currentToken.value += ch;
} else if (this.charTypes.carraigeReturn.test(ch)) {
this.state = states.CARRAIGE_RETURN;
} else if (this.charTypes.whitespace.test(ch)) {
this.currentToken.value += " ";
this.state = states.WHITESPACE;
} else {
this.emit('error', new Error('Unexpected character "' + ch + '" while parsing field body.'));
}
break;
case this.states.WHITESPACE:
if (this.charTypes.printableAscii.test(ch)) {
this.currentToken.value += ch;
this.state = states.FIELD_BODY;
} else if (this.charTypes.carraigeReturn.test(ch)) {
this.state = states.CARRAIGE_RETURN;
} else Eif (this.charTypes.whitespace.test(ch)) {
// Do nothing; just eat more whitespace
} else {
this.emit('error', new Error('Unexpected character "' + ch + '" found while parsing field body.'));
}
break;
case this.states.CARRAIGE_RETURN:
if (this.charTypes.newline.test(ch)) {
this.state = states.CRLF;
} else if (this.charTypes.whitespace.test(ch)) {
this.currentToken.value += " ";
this.state = states.WHITESPACE;
} else Eif (this.charTypes.printableAscii.test(ch)) {
// Push a space to account for the CR we read previously
this.currentToken.value += " ";
this.currentToken.value += ch;
this.state = states.FIELD_BODY;
} else {
this.emit('error', new Error('Unexpected character "' + ch + '" found while parsing field body.'));
}
break;
case this.states.CRLF:
if (this.charTypes.printableAscii.test(ch)) {
// A printable character on the very next line means we just
// finished the field body of one header and are now parsing
// the field name of a new header. Trim any whitespace from
// the beginning/end of the field body and emit.
this.currentToken.value = this.currentToken.value.trim();
this.emit('token', this.currentToken);
this.state = states.FIELD_NAME;
this.currentToken = {type: 'field_name', value: ch};
} else if (this.charTypes.carraigeReturn.test(ch)) {
this.state = states.CRLF_CR;
} else Eif (this.charTypes.whitespace.test(ch)) {
// This is just field body folding
this.currentToken.value += " ";
this.state = states.WHITESPACE;
} else {
this.emit('error', new Error('Unexpected character "' + ch + '" found while parsing field body.'));
}
break;
case this.states.CRLF_CR:
if (this.charTypes.newline.test(ch)) {
// We have already seen <CR><LF><CR>. Another <LF> means
// this is the empty line separating header fields from the
// message body. Emit the final field body token and a done
// event.
this.currentToken.value = this.currentToken.value.trim();
this.emit('token', this.currentToken);
this.emit('done');
this.state = states.DONE;
} else if (this.charTypes.whitespace.test(ch)) {
this.currentToken.value += " ";
this.state = states.WHITESPACE;
} else Eif (this.charTypes.printableAscii.test(ch)) {
// This is just field body folding
this.currentToken.value += " ";
this.currentToken.value += ch;
this.state = states.FIELD_BODY;
} else {
this.emit('error', new Error('Unexpected character "' + ch + '" found while parsing field body.'));
}
break;
case this.states.DONE:
// Trap state
break;
default:
// Should never get here, but in case we do...
this.emit('error', new Error('Unknown state "' + this.state + '"'));
}
};
module.exports = HeaderTokenizer;
|