1 | const { join, extname } = require('path')
|
2 | const fs = require('graceful-fs').promises
|
3 | const Hookable = require('hookable')
|
4 | const chokidar = require('chokidar')
|
5 | const JSON5 = require('json5')
|
6 | const Loki = require('@lokidb/loki').default
|
7 | const LokiFullTextSearch = require('@lokidb/full-text-search').default
|
8 | const logger = require('consola').withScope('@nuxt/content')
|
9 | const { default: PQueue } = require('p-queue')
|
10 | const { Markdown, YAML, CSV, XML } = require('../parsers')
|
11 |
|
12 | const QueryBuilder = require('./query-builder')
|
13 | const EXTENSIONS = ['.md', '.json', '.json5', '.yaml', '.yml', '.csv', '.xml']
|
14 |
|
15 | LokiFullTextSearch.register()
|
16 |
|
17 | class Database extends Hookable {
|
18 | constructor (options) {
|
19 | super()
|
20 | this.dir = options.dir || process.cwd()
|
21 | this.cwd = options.cwd || process.cwd()
|
22 | this.markdown = new Markdown(options.markdown)
|
23 | this.yaml = new YAML(options.yaml)
|
24 | this.csv = new CSV(options.csv)
|
25 | this.xml = new XML(options.xml)
|
26 |
|
27 | this.db = new Loki('content.db')
|
28 |
|
29 | this.items = this.db.addCollection('items', {
|
30 | fullTextSearch: options.fullTextSearchFields.map(field => ({ field })),
|
31 | nestedProperties: options.nestedProperties
|
32 | })
|
33 |
|
34 | this.extendParser = options.extendParser || {}
|
35 | this.extendParserExtensions = Object.keys(this.extendParser)
|
36 |
|
37 | options.watch && this.watch()
|
38 | this.options = options
|
39 | }
|
40 |
|
41 | |
42 |
|
43 |
|
44 |
|
45 |
|
46 | query (path, { deep = false, text = false } = {}) {
|
47 | const isDir = !path || !!this.dirs.find(dir => dir === path)
|
48 |
|
49 | const query = isDir ? { dir: deep ? { $regex: new RegExp(`^${path}`) } : path } : { path }
|
50 |
|
51 | const postprocess = isDir ? [] : [data => data[0]]
|
52 |
|
53 | return new QueryBuilder({
|
54 | query: this.items.chain().find(query, !isDir),
|
55 | path,
|
56 | postprocess,
|
57 | text
|
58 | }, this.options)
|
59 | }
|
60 |
|
61 | |
62 |
|
63 |
|
64 | async init () {
|
65 | this.dirs = ['/']
|
66 | this.items.clear()
|
67 |
|
68 | const startTime = process.hrtime()
|
69 | await this.walk(this.dir)
|
70 | const [s, ns] = process.hrtime(startTime)
|
71 | logger.info(`Parsed ${this.items.count()} files in ${s},${Math.round(ns / 1e8)} seconds`)
|
72 | }
|
73 |
|
74 | |
75 |
|
76 |
|
77 |
|
78 | async walk (dir) {
|
79 | let files = []
|
80 | try {
|
81 | files = await fs.readdir(dir)
|
82 | } catch (e) {
|
83 | logger.warn(`${dir} does not exist`)
|
84 | }
|
85 |
|
86 | await Promise.all(files.map(async (file) => {
|
87 | const path = join(dir, file)
|
88 | const stats = await fs.stat(path)
|
89 |
|
90 |
|
91 |
|
92 | if (file.includes('node_modules') || (/(^|\/)\.[^/.]/g).test(file)) {
|
93 | return
|
94 | }
|
95 |
|
96 |
|
97 | if (stats.isDirectory()) {
|
98 |
|
99 | this.dirs.push(this.normalizePath(path))
|
100 |
|
101 | return this.walk(path)
|
102 | } else if (stats.isFile()) {
|
103 |
|
104 | return this.insertFile(path)
|
105 | }
|
106 | }))
|
107 | }
|
108 |
|
109 | |
110 |
|
111 |
|
112 |
|
113 | async insertFile (path) {
|
114 | const items = await this.parseFile(path)
|
115 |
|
116 | if (!items) {
|
117 | return
|
118 | }
|
119 |
|
120 |
|
121 | if (items.length > 1) {
|
122 | this.dirs.push(this.normalizePath(path))
|
123 | }
|
124 |
|
125 | for (const item of items) {
|
126 | await this.callHook('file:beforeInsert', item)
|
127 |
|
128 | this.items.insert(item)
|
129 | }
|
130 | }
|
131 |
|
132 | |
133 |
|
134 |
|
135 |
|
136 | async updateFile (path) {
|
137 | const items = await this.parseFile(path)
|
138 |
|
139 | if (!items) {
|
140 | return
|
141 | }
|
142 |
|
143 | for (const item of items) {
|
144 | await this.callHook('file:beforeInsert', item)
|
145 |
|
146 | const document = this.items.findOne({ path: item.path })
|
147 |
|
148 | logger.info(`Updated ${path.replace(this.cwd, '.')}`)
|
149 | if (document) {
|
150 | this.items.update({ $loki: document.$loki, meta: document.meta, ...item })
|
151 | return
|
152 | }
|
153 | this.items.insert(item)
|
154 | }
|
155 | }
|
156 |
|
157 | |
158 |
|
159 |
|
160 |
|
161 | async removeFile (path) {
|
162 | const normalizedPath = await this.normalizePath(path)
|
163 | const document = this.items.findOne({ path: normalizedPath })
|
164 |
|
165 | this.items.remove(document)
|
166 | }
|
167 |
|
168 | |
169 |
|
170 |
|
171 |
|
172 | async parseFile (path) {
|
173 | const extension = extname(path)
|
174 |
|
175 | if (!EXTENSIONS.includes(extension) && !this.extendParserExtensions.includes(extension)) {
|
176 | return
|
177 | }
|
178 |
|
179 | const stats = await fs.stat(path)
|
180 | const file = {
|
181 | path,
|
182 | extension,
|
183 | data: await fs.readFile(path, 'utf-8')
|
184 | }
|
185 |
|
186 | await this.callHook('file:beforeParse', file)
|
187 |
|
188 |
|
189 | const parser = ({
|
190 | '.json': data => JSON.parse(data),
|
191 | '.json5': data => JSON5.parse(data),
|
192 | '.md': data => this.markdown.toJSON(data),
|
193 | '.csv': data => this.csv.toJSON(data),
|
194 | '.yaml': data => this.yaml.toJSON(data),
|
195 | '.yml': data => this.yaml.toJSON(data),
|
196 | '.xml': data => this.xml.toJSON(data),
|
197 | ...this.extendParser
|
198 | })[extension]
|
199 |
|
200 |
|
201 | let data = []
|
202 | try {
|
203 | data = await parser(file.data, { path: file.path })
|
204 |
|
205 | data = Array.isArray(data) ? data : [data]
|
206 | } catch (err) {
|
207 | logger.warn(`Could not parse ${path.replace(this.cwd, '.')}:`, err.message)
|
208 | return null
|
209 | }
|
210 |
|
211 |
|
212 | const normalizedPath = this.normalizePath(path)
|
213 |
|
214 |
|
215 | const isValidDate = (date) => {
|
216 | return date instanceof Date && !isNaN(date)
|
217 | }
|
218 |
|
219 | return data.map((item) => {
|
220 | const paths = normalizedPath.split('/')
|
221 |
|
222 | if (data.length > 1 && item.slug) {
|
223 | paths.push(item.slug)
|
224 | }
|
225 |
|
226 | const dir = paths.slice(0, paths.length - 1).join('/') || '/'
|
227 |
|
228 | const slug = paths[paths.length - 1]
|
229 |
|
230 | const path = paths.join('/')
|
231 |
|
232 |
|
233 | const existingCreatedAt = item.createdAt && new Date(item.createdAt)
|
234 | const existingUpdatedAt = item.updatedAt && new Date(item.updatedAt)
|
235 |
|
236 | return {
|
237 | slug,
|
238 |
|
239 | ...item,
|
240 | dir,
|
241 | path,
|
242 | extension,
|
243 | createdAt: isValidDate(existingCreatedAt) ? existingCreatedAt : stats.birthtime,
|
244 | updatedAt: isValidDate(existingUpdatedAt) ? existingUpdatedAt : stats.mtime
|
245 | }
|
246 | })
|
247 | }
|
248 |
|
249 | |
250 |
|
251 |
|
252 |
|
253 |
|
254 | normalizePath (path) {
|
255 | let extractPath = path.replace(this.dir, '')
|
256 | const extensionPath = extractPath.substr(extractPath.lastIndexOf('.'))
|
257 | const additionalsExt = EXTENSIONS.concat(this.extendParserExtensions)
|
258 |
|
259 |
|
260 | if (additionalsExt.includes(extensionPath) || extractPath.startsWith('.')) {
|
261 | extractPath = extractPath.replace(/(?:\.([^.]+))?$/, '')
|
262 | }
|
263 |
|
264 | return extractPath.replace(/\\/g, '/')
|
265 | }
|
266 |
|
267 | |
268 |
|
269 |
|
270 |
|
271 | watch () {
|
272 | this.queue = new PQueue({ concurrency: 1 })
|
273 |
|
274 | this.watcher = chokidar.watch(['**/*'], {
|
275 | cwd: this.dir,
|
276 | ignoreInitial: true,
|
277 | ignored: 'node_modules/**/*'
|
278 | })
|
279 | .on('change', path => this.queue.add(this.refresh.bind(this, 'change', path)))
|
280 | .on('add', path => this.queue.add(this.refresh.bind(this, 'add', path)))
|
281 | .on('unlink', path => this.queue.add(this.refresh.bind(this, 'unlink', path)))
|
282 | }
|
283 |
|
284 | |
285 |
|
286 |
|
287 |
|
288 | async refresh (event, path) {
|
289 | if (event === 'change') {
|
290 | await this.updateFile(`${this.dir}/${path}`)
|
291 | } else {
|
292 | await this.init()
|
293 | }
|
294 |
|
295 | this.callHook('file:updated', { event, path })
|
296 | }
|
297 |
|
298 | |
299 |
|
300 |
|
301 | async close () {
|
302 | await this.db.close()
|
303 | this.db = null
|
304 |
|
305 |
|
306 | if (this.watcher) {
|
307 | await this.watcher.close()
|
308 | this.watcher = null
|
309 | }
|
310 | }
|
311 | }
|
312 |
|
313 | module.exports = Database
|