1 | const tape = require('tape')
|
2 | const collect = require('stream-collector')
|
3 |
|
4 | const FuzzBuzz = require('fuzzbuzz')
|
5 | const create = require('./helpers/create')
|
6 |
|
7 | const MAX_PATH_DEPTH = 30
|
8 | const MAX_FILE_LENGTH = 1e3
|
9 | const CHARACTERS = 1e3
|
10 | const APPROX_READS_PER_FD = 5
|
11 | const APPROX_WRITES_PER_FD = 5
|
12 | const INVALID_CHARS = new Set(['/', '\\', '?', '%', '*', ':', '|', '"', '<', '>', '.', ' ', '\n', '\t', '\r'])
|
13 |
|
14 | class HyperdriveFuzzer extends FuzzBuzz {
|
15 | constructor (opts) {
|
16 | super(opts)
|
17 |
|
18 | this.add(10, this.writeFile)
|
19 | this.add(5, this.deleteFile)
|
20 | this.add(5, this.existingFileOverwrite)
|
21 | this.add(5, this.randomStatefulFileDescriptorRead)
|
22 | this.add(5, this.randomStatefulFileDescriptorWrite)
|
23 | this.add(3, this.statFile)
|
24 |
|
25 | this.add(2, this.deleteInvalidFile)
|
26 | this.add(2, this.randomReadStream)
|
27 | this.add(2, this.randomStatelessFileDescriptorRead)
|
28 | this.add(1, this.createReadableFileDescriptor)
|
29 |
|
30 | }
|
31 |
|
32 |
|
33 |
|
34 | _select (map) {
|
35 | let idx = this.randomInt(map.size - 1)
|
36 | if (idx < 0) return null
|
37 |
|
38 | let ite = map.entries()
|
39 | while (idx--) ite.next()
|
40 | return ite.next().value
|
41 | }
|
42 | _selectFile () {
|
43 | return this._select(this.files)
|
44 | }
|
45 | _selectDirectory () {
|
46 | return this._select(this.directories)
|
47 | }
|
48 | _selectReadableFileDescriptor () {
|
49 | return this._select(this.readable_fds)
|
50 | }
|
51 |
|
52 | _validChar () {
|
53 | do {
|
54 | var char = String.fromCharCode(this.randomInt(CHARACTERS))
|
55 | } while (INVALID_CHARS.has(char))
|
56 | return char
|
57 | }
|
58 | _fileName () {
|
59 | do {
|
60 | let depth = Math.max(this.randomInt(MAX_PATH_DEPTH), 1)
|
61 | var name = (new Array(depth)).fill(0).map(() => this._validChar()).join('/')
|
62 | } while (this.files.get(name) || this.directories.get(name))
|
63 | return name
|
64 | }
|
65 | _content () {
|
66 | return Buffer.allocUnsafe(this.randomInt(MAX_FILE_LENGTH)).fill(0).map(() => this.randomInt(10))
|
67 | }
|
68 | _createFile () {
|
69 | let name = this._fileName()
|
70 | let content = this._content()
|
71 | return { name, content }
|
72 | }
|
73 | _deleteFile (name) {
|
74 | return new Promise((resolve, reject) => {
|
75 | this.drive.unlink(name, err => {
|
76 | if (err) return reject(err)
|
77 | this.files.delete(name)
|
78 | return resolve({ type: 'delete', name })
|
79 | })
|
80 | })
|
81 | }
|
82 |
|
83 |
|
84 |
|
85 | _setup () {
|
86 | this.drive = create()
|
87 | this.files = new Map()
|
88 | this.directories = new Map()
|
89 | this.streams = new Map()
|
90 | this.readable_fds = new Map()
|
91 | this.log = []
|
92 |
|
93 | return new Promise((resolve, reject) => {
|
94 | this.drive.ready(err => {
|
95 | if (err) return reject(err)
|
96 | return resolve()
|
97 | })
|
98 | })
|
99 | }
|
100 |
|
101 | _validationDrive () {
|
102 | return this.drive
|
103 | }
|
104 | _validateFile (name, content) {
|
105 | let drive = this._validationDrive()
|
106 | return new Promise((resolve, reject) => {
|
107 | drive.readFile(name, (err, data) => {
|
108 | if (err) return reject(err)
|
109 | if (!data.equals(content)) return reject(new Error(`Read data for ${name} does not match written content.`))
|
110 | return resolve()
|
111 | })
|
112 | })
|
113 | }
|
114 | _validateDirectory (name, list) {
|
115 | |
116 |
|
117 |
|
118 |
|
119 |
|
120 |
|
121 |
|
122 |
|
123 |
|
124 |
|
125 |
|
126 |
|
127 |
|
128 |
|
129 |
|
130 | }
|
131 | async _validate () {
|
132 | for (const [fileName, content] of this.files) {
|
133 | await this._validateFile(fileName, content)
|
134 | }
|
135 | for (const [dirName, list] of this.directories) {
|
136 | await this._validateDirectory(dirName, list)
|
137 | }
|
138 | }
|
139 |
|
140 | async call (ops) {
|
141 | let res = await super.call(ops)
|
142 | this.log.push(res)
|
143 | }
|
144 |
|
145 |
|
146 |
|
147 | writeFile () {
|
148 | let { name, content } = this._createFile()
|
149 | return new Promise((resolve, reject) => {
|
150 | this.debug(`Writing file ${name} with content ${content.length}`)
|
151 | this.drive.writeFile(name, content, err => {
|
152 | if (err) return reject(err)
|
153 | this.files.set(name, content)
|
154 | return resolve({ type: 'write', name, content })
|
155 | })
|
156 | })
|
157 | }
|
158 |
|
159 | deleteFile () {
|
160 | let selected = this._selectFile()
|
161 | if (!selected) return
|
162 |
|
163 | let fileName = selected[0]
|
164 |
|
165 | this.debug(`Deleting valid file: ${fileName}`)
|
166 |
|
167 | return this._deleteFile(fileName)
|
168 | }
|
169 |
|
170 | async deleteInvalidFile () {
|
171 | let name = this._fileName()
|
172 | while (this.files.get(name)) name = this._fileName()
|
173 | try {
|
174 | this.debug(`Deleting invalid file: ${name}`)
|
175 | await this._deleteFile(name)
|
176 | } catch (err) {
|
177 | if (err && err.code !== 'ENOENT') throw err
|
178 | }
|
179 | }
|
180 |
|
181 | statFile () {
|
182 | let selected = this._selectFile()
|
183 | if (!selected) return
|
184 |
|
185 | let [fileName, content] = selected
|
186 | return new Promise((resolve, reject) => {
|
187 | this.debug(`Statting file: ${fileName}`)
|
188 | this.drive.stat(fileName, (err, st) => {
|
189 | if (err) return reject(err)
|
190 | if (!st) return reject(new Error(`File ${fileName} should exist but does not exist.`))
|
191 | if (st.size !== content.length) return reject(new Error(`Incorrect content length for file ${fileName}.`))
|
192 | return resolve({ type: 'stat', fileName, stat: st })
|
193 | })
|
194 | })
|
195 | }
|
196 |
|
197 | statDirectory () {
|
198 | let selected = this._selectDirectory()
|
199 | if (!selected) return
|
200 |
|
201 | let [dirName, { offset, byteOffset }] = selected
|
202 |
|
203 | this.debug(`Statting directory ${dirName}.`)
|
204 | return new Promise((resolve, reject) => {
|
205 | this.drive.stat(dirName, (err, st) => {
|
206 | if (err) return reject(err)
|
207 | if (!st) return reject(new Error(`Directory ${dirName} should exist but does not exist.`))
|
208 | if (!st.isDirectory()) return reject(new Error(`Stat for directory ${dirName} does not have directory mode`))
|
209 | console.log('st:', st, 'offset:', offset, 'byteOffset:', byteOffset)
|
210 | if (st.offset !== offset || st.byteOffset !== byteOffset) return reject(new Error(`Invalid offsets for ${dirName}`))
|
211 | this.debug(` Successfully statted directory.`)
|
212 | return resolve({ type: 'stat', dirName })
|
213 | })
|
214 | })
|
215 | }
|
216 |
|
217 | existingFileOverwrite () {
|
218 | let selected = this._selectFile()
|
219 | if (!selected) return
|
220 | let [fileName] = selected
|
221 |
|
222 | let { content: newContent } = this._createFile()
|
223 |
|
224 | return new Promise((resolve, reject) => {
|
225 | this.debug(`Overwriting existing file: ${fileName}`)
|
226 | let writeStream = this.drive.createWriteStream(fileName)
|
227 | writeStream.on('error', err => reject(err))
|
228 | writeStream.on('finish', () => {
|
229 | this.files.set(fileName, newContent)
|
230 | resolve()
|
231 | })
|
232 | writeStream.end(newContent)
|
233 | })
|
234 | }
|
235 |
|
236 | randomReadStream () {
|
237 | let selected = this._selectFile()
|
238 | if (!selected) return
|
239 | let [fileName, content] = selected
|
240 |
|
241 | return new Promise((resolve, reject) => {
|
242 | let drive = this._validationDrive()
|
243 | let start = this.randomInt(content.length)
|
244 | let length = this.randomInt(content.length - start)
|
245 | this.debug(`Creating random read stream for ${fileName} at start ${start} with length ${length}`)
|
246 | let stream = drive.createReadStream(fileName, {
|
247 | start,
|
248 | length
|
249 | })
|
250 | collect(stream, (err, bufs) => {
|
251 | if (err) return reject(err)
|
252 | let buf = bufs.length === 1 ? bufs[0] : Buffer.concat(bufs)
|
253 |
|
254 | if (!buf.equals(content.slice(start, start + length))) {
|
255 | console.log('buf:', buf, 'content slice:', content.slice(start, start + length))
|
256 | return reject(new Error('Read stream does not match content slice.'))
|
257 | }
|
258 | this.debug(`Random read stream for ${fileName} succeeded.`)
|
259 | return resolve()
|
260 | })
|
261 | })
|
262 | }
|
263 |
|
264 | randomStatelessFileDescriptorRead () {
|
265 | let selected = this._selectFile()
|
266 | if (!selected) return
|
267 | let [fileName, content] = selected
|
268 |
|
269 | let length = this.randomInt(content.length)
|
270 | let start = this.randomInt(content.length)
|
271 | let actualLength = Math.min(length, content.length)
|
272 | let buf = Buffer.alloc(actualLength)
|
273 |
|
274 | return new Promise((resolve, reject) => {
|
275 | let drive = this._validationDrive()
|
276 | this.debug(`Random stateless file descriptor read for ${fileName}, ${length} starting at ${start}`)
|
277 | drive.open(fileName, 'r', (err, fd) => {
|
278 | if (err) return reject(err)
|
279 |
|
280 | drive.read(fd, buf, 0, length, start, (err, bytesRead) => {
|
281 | if (err) return reject(err)
|
282 | buf = buf.slice(0, bytesRead)
|
283 | let expected = content.slice(start, start + bytesRead)
|
284 | if (!buf.equals(expected)) return reject(new Error('File descriptor read does not match slice.'))
|
285 | drive.close(fd, err => {
|
286 | if (err) return reject(err)
|
287 |
|
288 | this.debug(`Random file descriptor read for ${fileName} succeeded`)
|
289 |
|
290 | return resolve()
|
291 | })
|
292 | })
|
293 | })
|
294 | })
|
295 | }
|
296 |
|
297 | createReadableFileDescriptor () {
|
298 | let selected = this._selectFile()
|
299 | if (!selected) return
|
300 | let [fileName, content] = selected
|
301 |
|
302 | let start = this.randomInt(content.length / 5)
|
303 | let drive = this._validationDrive()
|
304 |
|
305 | return new Promise((resolve, reject) => {
|
306 | this.debug(`Creating readable FD for file ${fileName} and start: ${start}`)
|
307 | drive.open(fileName, 'r', (err, fd) => {
|
308 | if (err) return reject(err)
|
309 | this.readable_fds.set(fd, {
|
310 | pos: start,
|
311 | started: false,
|
312 | content
|
313 | })
|
314 | return resolve()
|
315 | })
|
316 | })
|
317 | }
|
318 |
|
319 | randomStatefulFileDescriptorRead () {
|
320 | let selected = this._selectReadableFileDescriptor()
|
321 | if (!selected) return
|
322 | let [fd, fdInfo] = selected
|
323 |
|
324 | let { content, pos, started } = fdInfo
|
325 |
|
326 | let length = this.randomInt(content.length / APPROX_READS_PER_FD)
|
327 | let actualLength = Math.min(length, content.length)
|
328 | let buf = Buffer.alloc(actualLength)
|
329 |
|
330 | this.debug(`Reading from random stateful FD ${fd}`)
|
331 |
|
332 | let self = this
|
333 |
|
334 | return new Promise((resolve, reject) => {
|
335 | let drive = this._validationDrive()
|
336 |
|
337 | let start = null
|
338 | if (!started) {
|
339 | fdInfo.started = true
|
340 | start = fdInfo.pos
|
341 | }
|
342 |
|
343 | drive.read(fd, buf, 0, length, start, (err, bytesRead) => {
|
344 | if (err) return reject(err)
|
345 |
|
346 | if (!bytesRead && length) {
|
347 | return close()
|
348 | }
|
349 |
|
350 | buf = buf.slice(0, bytesRead)
|
351 | let expected = content.slice(pos, pos + bytesRead)
|
352 | if (!buf.equals(expected)) return reject(new Error('File descriptor read does not match slice.'))
|
353 |
|
354 | fdInfo.pos += bytesRead
|
355 | return resolve()
|
356 | })
|
357 |
|
358 | function close () {
|
359 | drive.close(fd, err => {
|
360 | if (err) return reject(err)
|
361 | self.readable_fds.delete(fd)
|
362 | return resolve()
|
363 | })
|
364 | }
|
365 | })
|
366 | }
|
367 |
|
368 | randomStatefulFileDescriptorWrite () {
|
369 | let append = !!this.randomInt(1)
|
370 | let flags = append ? 'a' : 'w+'
|
371 |
|
372 | if (append) {
|
373 | let selected = this._selectFile()
|
374 | if (!selected) return
|
375 | var [fileName, content] = selected
|
376 | var pos = content.length
|
377 | } else {
|
378 | fileName = this._fileName()
|
379 | content = Buffer.alloc(0)
|
380 | pos = 0
|
381 | }
|
382 |
|
383 | const bufs = new Array(this.randomInt(APPROX_WRITES_PER_FD - 1)).fill(0).map(() => this._content())
|
384 | const self = this
|
385 |
|
386 | let count = 0
|
387 |
|
388 | return new Promise((resolve, reject) => {
|
389 | this.debug(`Writing stateful file descriptor for fileName ${fileName} with flags ${flags} and buffers ${bufs.length}`)
|
390 | this.drive.open(fileName, flags, (err, fd) => {
|
391 | if (err) return reject(err)
|
392 | if (!bufs.length) return close(fd)
|
393 | return writeNext(fd)
|
394 | })
|
395 |
|
396 | function writeNext (fd) {
|
397 | let next = bufs[count]
|
398 | self.debug(` Writing content with length ${next.length} to FD ${fd} at pos: ${pos}`)
|
399 | self.drive.write(fd, next, 0, next.length, pos, (err, bytesWritten) => {
|
400 | if (err) return reject(err)
|
401 | pos += bytesWritten
|
402 | bufs[count] = next.slice(0, bytesWritten)
|
403 | if (++count === bufs.length) return close(fd)
|
404 | return writeNext(fd)
|
405 | })
|
406 | }
|
407 |
|
408 | function close (fd) {
|
409 | self.drive.close(fd, err => {
|
410 | if (err) return reject(err)
|
411 | self.files.set(fileName, Buffer.concat([content, ...bufs]))
|
412 | return resolve()
|
413 | })
|
414 | }
|
415 | })
|
416 | }
|
417 |
|
418 | writeAndMkdir () {
|
419 | const self = this
|
420 |
|
421 | let { name: fileName, content } = this._createFile()
|
422 | let dirName = this._fileName()
|
423 |
|
424 | return new Promise((resolve, reject) => {
|
425 | this.debug(`Writing ${fileName} and making dir ${dirName} simultaneously`)
|
426 |
|
427 | let pending = 2
|
428 |
|
429 | let offset = this.drive._contentFeedLength
|
430 | let byteOffset = this.drive._contentFeedByteLength
|
431 |
|
432 | let writeStream = this.drive.createWriteStream(fileName)
|
433 | writeStream.on('finish', done)
|
434 |
|
435 | this.drive.mkdir(dirName, done)
|
436 | writeStream.end(content)
|
437 |
|
438 | function done (err) {
|
439 | if (err) return reject(err)
|
440 | if (!--pending) {
|
441 | self.files.set(fileName, content)
|
442 | self.debug(`Created directory ${dirName}`)
|
443 | self.directories.set(dirName, {
|
444 | offset,
|
445 | byteOffset
|
446 | })
|
447 | return resolve()
|
448 | }
|
449 | }
|
450 | })
|
451 | }
|
452 | }
|
453 |
|
454 | class SparseHyperdriveFuzzer extends HyperdriveFuzzer {
|
455 | async _setup () {
|
456 | await super._setup()
|
457 |
|
458 | this.remoteDrive = create(this.drive.key, { sparse: true })
|
459 |
|
460 | return new Promise((resolve, reject) => {
|
461 | this.remoteDrive.ready(err => {
|
462 | if (err) throw err
|
463 | let s1 = this.remoteDrive.replicate(true, { live: true, timeout: 0 })
|
464 | s1.pipe(this.drive.replicate(false, { live: true, timeout: 0 })).pipe(s1)
|
465 | this.remoteDrive.ready(err => {
|
466 | if (err) return reject(err)
|
467 | return resolve()
|
468 | })
|
469 | })
|
470 | })
|
471 | }
|
472 | _validationDrive () {
|
473 | return this.remoteDrive
|
474 | }
|
475 | }
|
476 |
|
477 | module.exports = HyperdriveFuzzer
|
478 |
|
479 | tape('20000 mixed operations, single drive', async t => {
|
480 | t.plan(1)
|
481 |
|
482 | const fuzz = new HyperdriveFuzzer({
|
483 | seed: 'hyperdrive',
|
484 | debugging: false
|
485 | })
|
486 |
|
487 | try {
|
488 | await fuzz.run(20000)
|
489 | t.pass('fuzzing succeeded')
|
490 | } catch (err) {
|
491 | t.error(err, 'no error')
|
492 | }
|
493 | })
|
494 |
|
495 | tape('20000 mixed operations, replicating drives', async t => {
|
496 | t.plan(1)
|
497 |
|
498 | const fuzz = new SparseHyperdriveFuzzer({
|
499 | seed: 'hyperdrive2',
|
500 | debugging: false
|
501 | })
|
502 |
|
503 | try {
|
504 | await fuzz.run(20000)
|
505 | t.pass('fuzzing succeeded')
|
506 | } catch (err) {
|
507 | t.error(err, 'no error')
|
508 | }
|
509 | })
|
510 |
|
511 | tape('100 quick validations (initialization timing)', async t => {
|
512 | t.plan(1)
|
513 |
|
514 | try {
|
515 | for (let i = 0; i < 100; i++) {
|
516 | const fuzz = new HyperdriveFuzzer({
|
517 | seed: 'iteration #' + i,
|
518 | debugging: false
|
519 | })
|
520 | await fuzz.run(100)
|
521 | }
|
522 | t.pass('fuzzing suceeded')
|
523 | } catch (err) {
|
524 | t.error(err, 'no error')
|
525 | }
|
526 | })
|