UNPKG

9.13 kBJavaScriptView Raw
1const { join, extname } = require('path')
2const fs = require('graceful-fs').promises
3const Hookable = require('hookable')
4const chokidar = require('chokidar')
5const JSON5 = require('json5')
6const Loki = require('@lokidb/loki').default
7const LokiFullTextSearch = require('@lokidb/full-text-search').default
8const logger = require('consola').withScope('@nuxt/content')
9const { default: PQueue } = require('p-queue')
10const { Markdown, YAML, CSV, XML } = require('../parsers')
11
12const QueryBuilder = require('./query-builder')
13const EXTENSIONS = ['.md', '.json', '.json5', '.yaml', '.yml', '.csv', '.xml']
14
15LokiFullTextSearch.register()
16
17class Database extends Hookable {
18 constructor (options) {
19 super()
20 this.dir = options.dir || process.cwd()
21 this.cwd = options.cwd || process.cwd()
22 this.markdown = new Markdown(options.markdown)
23 this.yaml = new YAML(options.yaml)
24 this.csv = new CSV(options.csv)
25 this.xml = new XML(options.xml)
26 // Create Loki database
27 this.db = new Loki('content.db')
28 // Init collection
29 this.items = this.db.addCollection('items', {
30 fullTextSearch: options.fullTextSearchFields.map(field => ({ field })),
31 nestedProperties: options.nestedProperties
32 })
33 // User Parsers
34 this.extendParser = options.extendParser || {}
35 this.extendParserExtensions = Object.keys(this.extendParser)
36 // Call chokidar watch if option if provided (dev only)
37 options.watch && this.watch()
38 this.options = options
39 }
40
41 /**
42 * Query items from collection
43 * @param {string} path - Requested path (path / directory).
44 * @returns {QueryBuilder} Instance of QueryBuilder to be chained
45 */
46 query (path, { deep = false, text = false } = {}) {
47 const isDir = !path || !!this.dirs.find(dir => dir === path)
48 // Look for dir or path
49 const query = isDir ? { dir: deep ? { $regex: new RegExp(`^${path}`) } : path } : { path }
50 // Postprocess to get only first result (findOne)
51 const postprocess = isDir ? [] : [data => data[0]]
52
53 return new QueryBuilder({
54 query: this.items.chain().find(query, !isDir),
55 path,
56 postprocess,
57 text
58 }, this.options)
59 }
60
61 /**
62 * Clear items in database and load files into collection
63 */
64 async init () {
65 this.dirs = ['/']
66 this.items.clear()
67
68 const startTime = process.hrtime()
69 await this.walk(this.dir)
70 const [s, ns] = process.hrtime(startTime)
71 logger.info(`Parsed ${this.items.count()} files in ${s},${Math.round(ns / 1e8)} seconds`)
72 }
73
74 /**
75 * Walk dir tree recursively
76 * @param {string} dir - Directory to browse.
77 */
78 async walk (dir) {
79 let files = []
80 try {
81 files = await fs.readdir(dir)
82 } catch (e) {
83 logger.warn(`${dir} does not exist`)
84 }
85
86 await Promise.all(files.map(async (file) => {
87 const path = join(dir, file)
88 const stats = await fs.stat(path)
89
90 // ignore node_modules or hidden file
91 /* istanbul ignore if */
92 if (file.includes('node_modules') || (/(^|\/)\.[^/.]/g).test(file)) {
93 return
94 }
95
96 /* istanbul ignore else */
97 if (stats.isDirectory()) {
98 // Store directory in local variable to be checked later
99 this.dirs.push(this.normalizePath(path))
100 // Walk recursively subfolder
101 return this.walk(path)
102 } else if (stats.isFile()) {
103 // Add file to collection
104 return this.insertFile(path)
105 }
106 }))
107 }
108
109 /**
110 * Insert file in collection
111 * @param {string} path - The path of the file.
112 */
113 async insertFile (path) {
114 const items = await this.parseFile(path)
115
116 if (!items) {
117 return
118 }
119
120 // Assume path is a directory if returning an array
121 if (items.length > 1) {
122 this.dirs.push(this.normalizePath(path))
123 }
124
125 for (const item of items) {
126 await this.callHook('file:beforeInsert', item)
127
128 this.items.insert(item)
129 }
130 }
131
132 /**
133 * Update file in collection
134 * @param {string} path - The path of the file.
135 */
136 async updateFile (path) {
137 const items = await this.parseFile(path)
138
139 if (!items) {
140 return
141 }
142
143 for (const item of items) {
144 await this.callHook('file:beforeInsert', item)
145
146 const document = this.items.findOne({ path: item.path })
147
148 logger.info(`Updated ${path.replace(this.cwd, '.')}`)
149 if (document) {
150 this.items.update({ $loki: document.$loki, meta: document.meta, ...item })
151 return
152 }
153 this.items.insert(item)
154 }
155 }
156
157 /**
158 * Remove file from collection
159 * @param {string} path - The path of the file.
160 */
161 async removeFile (path) {
162 const normalizedPath = await this.normalizePath(path)
163 const document = this.items.findOne({ path: normalizedPath })
164
165 this.items.remove(document)
166 }
167
168 /**
169 * Read a file and transform it to be insert / updated in collection
170 * @param {string} path - The path of the file.
171 */
172 async parseFile (path) {
173 const extension = extname(path)
174 // If unkown extension, skip
175 if (!EXTENSIONS.includes(extension) && !this.extendParserExtensions.includes(extension)) {
176 return
177 }
178
179 const stats = await fs.stat(path)
180 const file = {
181 path,
182 extension,
183 data: await fs.readFile(path, 'utf-8')
184 }
185
186 await this.callHook('file:beforeParse', file)
187
188 // Get parser depending on extension
189 const parser = ({
190 '.json': data => JSON.parse(data),
191 '.json5': data => JSON5.parse(data),
192 '.md': data => this.markdown.toJSON(data),
193 '.csv': data => this.csv.toJSON(data),
194 '.yaml': data => this.yaml.toJSON(data),
195 '.yml': data => this.yaml.toJSON(data),
196 '.xml': data => this.xml.toJSON(data),
197 ...this.extendParser
198 })[extension]
199
200 // Collect data from file
201 let data = []
202 try {
203 data = await parser(file.data, { path: file.path })
204 // Force data to be an array
205 data = Array.isArray(data) ? data : [data]
206 } catch (err) {
207 logger.warn(`Could not parse ${path.replace(this.cwd, '.')}:`, err.message)
208 return null
209 }
210
211 // Normalize path without dir and ext
212 const normalizedPath = this.normalizePath(path)
213
214 // Validate the existing dates to avoid wrong date format or typo
215 const isValidDate = (date) => {
216 return date instanceof Date && !isNaN(date)
217 }
218
219 return data.map((item) => {
220 const paths = normalizedPath.split('/')
221 // `item.slug` is necessary with JSON arrays since `slug` comes from filename by default
222 if (data.length > 1 && item.slug) {
223 paths.push(item.slug)
224 }
225 // Extract `dir` from paths
226 const dir = paths.slice(0, paths.length - 1).join('/') || '/'
227 // Extract `slug` from paths
228 const slug = paths[paths.length - 1]
229 // Construct full path
230 const path = paths.join('/')
231
232 // Overrides createdAt & updatedAt if it exists in the document
233 const existingCreatedAt = item.createdAt && new Date(item.createdAt)
234 const existingUpdatedAt = item.updatedAt && new Date(item.updatedAt)
235
236 return {
237 slug,
238 // Allow slug override
239 ...item,
240 dir,
241 path,
242 extension,
243 createdAt: isValidDate(existingCreatedAt) ? existingCreatedAt : stats.birthtime,
244 updatedAt: isValidDate(existingUpdatedAt) ? existingUpdatedAt : stats.mtime
245 }
246 })
247 }
248
249 /**
250 * Remove base dir and extension from file path
251 * @param {string} path - The path of the file.
252 * @returns {string} Normalized path
253 */
254 normalizePath (path) {
255 let extractPath = path.replace(this.dir, '')
256 const extensionPath = extractPath.substr(extractPath.lastIndexOf('.'))
257 const additionalsExt = EXTENSIONS.concat(this.extendParserExtensions)
258
259 // Remove the extension from the path if contained at the end or starts with a dot
260 if (additionalsExt.includes(extensionPath) || extractPath.startsWith('.')) {
261 extractPath = extractPath.replace(/(?:\.([^.]+))?$/, '')
262 }
263
264 return extractPath.replace(/\\/g, '/')
265 }
266
267 /**
268 * Watch base dir for changes
269 */
270 /* istanbul ignore next */
271 watch () {
272 this.queue = new PQueue({ concurrency: 1 })
273
274 this.watcher = chokidar.watch(['**/*'], {
275 cwd: this.dir,
276 ignoreInitial: true,
277 ignored: 'node_modules/**/*'
278 })
279 .on('change', path => this.queue.add(this.refresh.bind(this, 'change', path)))
280 .on('add', path => this.queue.add(this.refresh.bind(this, 'add', path)))
281 .on('unlink', path => this.queue.add(this.refresh.bind(this, 'unlink', path)))
282 }
283
284 /**
285 * Init database and broadcast change through Websockets
286 */
287 /* istanbul ignore next */
288 async refresh (event, path) {
289 if (event === 'change') {
290 await this.updateFile(`${this.dir}/${path}`)
291 } else {
292 await this.init()
293 }
294
295 this.callHook('file:updated', { event, path })
296 }
297
298 /*
299 ** Stop database and watcher and clear pointers
300 */
301 async close () {
302 await this.db.close()
303 this.db = null
304
305 /* istanbul ignore if */
306 if (this.watcher) {
307 await this.watcher.close()
308 this.watcher = null
309 }
310 }
311}
312
313module.exports = Database