1 | /*
|
2 | This Transport layers uses GUN.
|
3 |
|
4 | See https://github.com/internetarchive/dweb-mirror/issues/43 for meta issue
|
5 | */
|
6 | const Url = require('url');
|
7 | process.env.GUN_ENV = "false";
|
8 |
|
9 | /* This should be done in the caller (see dweb-archive/archive.html for example)
|
10 | const Gun = require('gun/gun.js'); // gun/gun is the minimized version
|
11 | // Raw Gun has almost nothing in it, it needs at least the following to work properly.
|
12 | require('gun/lib/path.js'); // So that .path works
|
13 |
|
14 | //WORKAROUND-GUN-STORAGE
|
15 | // The next step is to stop it failing as soon as its cached 5Mb in localstorage
|
16 | // see https://github.com/amark/gun/blob/master/test/tmp/indexedDB.html and https://github.com/amark/gun/issues/590
|
17 | // but the instructions on how to do this are obviously broken so waiting on @amark to get this working.
|
18 |
|
19 | // See https://github.com/internetarchive/dweb-archive/issues/106 unable to get this working (Gun doesnt work well with webpack)
|
20 | //require('gun/nts');
|
21 | //require('gun/lib/wire'); // NodeJS websocket
|
22 | //require('gun/lib/multicast'); // Local area broadcasting needs passing `multicast: true` in options (which is safe in node + browser)
|
23 | require('gun/lib/radix.js'); // loaded by store but required for webpack
|
24 | require('gun/lib/radisk.js'); // loaded by store but required for webpack
|
25 | require('gun/lib/store.js');
|
26 | require('gun/lib/rindexed.js');
|
27 | */
|
28 | const debuggun = require('debug')('dweb-transports:gun');
|
29 | const canonicaljson = require('@stratumn/canonicaljson');
|
30 |
|
31 | // Other Dweb modules
|
32 | const errors = require('./Errors'); // Standard Dweb Errors
|
33 | const Transport = require('./Transport.js'); // Base class for TransportXyz
|
34 | const Transports = require('./Transports'); // Manage all Transports that are loaded
|
35 | const utils = require('./utils'); // Utility functions
|
36 |
|
37 | // Utility packages (ours) And one-liners
|
38 | //unused currently: function delay(ms, val) { return new Promise(resolve => {setTimeout(() => { resolve(val); },ms)})}
|
39 |
|
40 | let defaultoptions = {
|
41 | peers: [ "https://dweb.me:4246/gun" ],
|
42 | localStorage: false // Need to be false to turn localStorage off on browser (do this if include radix/radisk)
|
43 | };
|
44 | //To run a superpeer - cd wherever; node install gun; cd node_modules/gun; npm start - starts server by default on port 8080, or set an "env" - see http.js
|
45 | //setenv GUN_ENV false; node examples/http.js 4246
|
46 | //Make sure to open of the port (typically in /etc/ferm)
|
47 | // TODO-GUN - copy example from systemctl here
|
48 |
|
49 | /*
|
50 | WORKING AROUND GUN WEIRDNESS/SUBOPTIMAL (of course, whats weird/sub-optimal to me, might be ideal to someone else) - search the code to see where worked around
|
51 |
|
52 | WORKAROUND-GUN-UNDERSCORE .once() and possibly .on() send an extra GUN internal field "_" which needs filtering. Reported and hopefully will get fixed
|
53 | .once behaves differently on node or the browser - this is a bug https://github.com/amark/gun/issues/586 and for now this code doesnt work on Node
|
54 | WORKAROUND-GUN-CURRENT: .once() and .on() deliver existing values as well as changes, reported & hopefully will get way to find just new ones.
|
55 | WORKAROUND-GUN-DELETE: There is no way to delete an item, setting it to null is recorded and is by convention a deletion. BUT the field will still show up in .once and .on,
|
56 | WORKAROUND-GUN-PROMISES: GUN is not promisified, there is only one place we care, and that is .once (since .on is called multiple times).
|
57 | WORKAROUND-GUN-ERRORS: GUN does an unhelpful job with errors, for example returning undefined when it cant find something (e.g. if connection to superpeer is down),
|
58 | for now just throw an error on undefined
|
59 | WORKAROUND-GUN-STORAGE: GUN defaults to local storage, which then fails on 5Mb or more of data, need to use radix, which has to be included and has bizarre config requirement I can't figure out
|
60 | TODO-GUN, handle error callbacks which are available in put etc
|
61 | Errors and Promises: Note that GUN's use of promises is seriously unexpected (aka weird), see https://gun.eco/docs/SEA#errors
|
62 | instead of using .reject or throwing an error at async it puts the error in SEA.err, so how that works in async parallel context is anyone's guess
|
63 | */
|
64 |
|
65 | class TransportGUN extends Transport {
|
66 | /*
|
67 | GUN specific transport - over IPFS
|
68 |
|
69 | Fields:
|
70 | gun: object returned when starting GUN
|
71 | */
|
72 |
|
73 | constructor(options) {
|
74 | super(options);
|
75 | this.options = options; // Dictionary of options
|
76 | this.gun = undefined;
|
77 | this.name = "GUN"; // For console log etc
|
78 | this.supportURLs = ['gun'];
|
79 | this.supportFunctions = [ 'fetch', //'store'
|
80 | 'connection', 'get', 'set', 'getall', 'keys', 'newdatabase', 'newtable', 'monitor',
|
81 | 'add', 'list', 'listmonitor', 'newlisturls'];
|
82 | this.supportFeatures = []; // Doesnt support noCache and is mutable
|
83 | this.status = Transport.STATUS_LOADED;
|
84 | }
|
85 |
|
86 | connection(url) {
|
87 | /*
|
88 | TODO-GUN need to determine what a "rooted" Url is in gun, is it specific to a superpeer for example
|
89 | Utility function to get Gun object for this URL (note this isn't async)
|
90 | url: URL string or structure, to find list of of form [gun|dweb]:/gun/<database>/<table>[/<key] but could be arbitrary gun path
|
91 | resolves: Gun a connection to use for get's etc, undefined if fails
|
92 | */
|
93 | url = Url.parse(url); // Accept string or Url structure
|
94 | let patharray = url.pathname.split('/'); //[ 'gun', database, table ] but could be arbitrary length path
|
95 | patharray.shift(); // Loose leading ""
|
96 | patharray.shift(); // Loose "gun"
|
97 | debuggun("path=", patharray);
|
98 | return this.gun.path(patharray); // Not sure how this could become undefined as it will return g before the path is walked, but if do a lookup on this "g" then should get undefined
|
99 | }
|
100 |
|
101 | //TODO-SPLIT define load()
|
102 |
|
103 | static setup0(options) {
|
104 | /*
|
105 | First part of setup, create obj, add to Transports but dont attempt to connect, typically called instead of p_setup if want to parallelize connections.
|
106 | options: { gun: { }, } Set of options - "gun" is used for those to pass direct to Gun
|
107 | */
|
108 | let combinedoptions = Transport.mergeoptions(defaultoptions, options.gun);
|
109 | debuggun("options %o", combinedoptions);
|
110 | let t = new TransportGUN(combinedoptions); // Note doesnt start IPFS or OrbitDB
|
111 | t.gun = new Gun(t.options); // This doesnt connect, just creates db structure
|
112 | Transports.addtransport(t);
|
113 | return t;
|
114 | }
|
115 |
|
116 | async p_setup1(cb) {
|
117 | /*
|
118 | This sets up for GUN.
|
119 | Throws: TODO-GUN-DOC document possible error behavior
|
120 | */
|
121 | try {
|
122 | this.status = Transport.STATUS_STARTING; // Should display, but probably not refreshed in most case
|
123 | if (cb) cb(this);
|
124 | //TODO-GUN-TEST - try connect and retrieve info then look at ._.opt.peers
|
125 | await this.p_status();
|
126 | } catch(err) {
|
127 | console.error(this.name,"failed to start",err);
|
128 | this.status = Transport.STATUS_FAILED;
|
129 | }
|
130 | if (cb) cb(this);
|
131 | return this;
|
132 | }
|
133 |
|
134 | async p_status() {
|
135 | /*
|
136 | Return an integer for the status of a transport see Transport
|
137 | */
|
138 | //TODO-GUN-TEST - try connect and retrieve info then look at ._.opt.peers
|
139 | this.status = Transport.STATUS_CONNECTED; //TODO-GUN how do I know if/when I'm connected (see comment on p_setup1 as well)
|
140 | return this.status;
|
141 | }
|
142 | // ===== DATA ======
|
143 |
|
144 | async p_rawfetch(url) {
|
145 | url = Url.parse(url); // Accept url as string or object
|
146 | let g = this.connection(url); // Goes all the way to the key
|
147 | let val = await this._p_once(g);
|
148 | //g.on((data)=>debuggun("Got late result of: %o", data)); // Checking for bug in GUN issue#586 - ignoring result
|
149 | if (!val)
|
150 | throw new errors.TransportError("GUN unable to retrieve: "+url.href); // WORKAROUND-GUN-ERRORS - gun doesnt throw errors when it cant find something
|
151 | let o = typeof val === "string" ? JSON.parse(val) : val; // This looks like it is sync (see same code on p_get and p_rawfetch)
|
152 | //TODO-GUN this is a hack because the metadata such as metadata/audio is getting cached in GUN and in this case is wrong.
|
153 | if (o.metadata && o.metadata.thumbnaillinks && o.metadata.thumbnaillinks.find(t => t.includes('ipfs/zb2'))) {
|
154 | throw new errors.TransportError("GUN retrieving legacy data at: "+url.href)
|
155 | }
|
156 | return o;
|
157 | }
|
158 |
|
159 |
|
160 | // ===== LISTS ========
|
161 |
|
162 | // noinspection JSCheckFunctionSignatures
|
163 | async p_rawlist(url) {
|
164 | /*
|
165 | Fetch all the objects in a list, these are identified by the url of the public key used for signing.
|
166 | (Note this is the 'signedby' parameter of the p_rawadd call, not the 'url' parameter
|
167 | Returns a promise that resolves to the list.
|
168 | Each item of the list is a dict: {"url": url, "date": date, "signature": signature, "signedby": signedby}
|
169 | List items may have other data (e.g. reference ids of underlying transport)
|
170 |
|
171 | :param string url: String with the url that identifies the list.
|
172 | :resolve array: An array of objects as stored on the list.
|
173 | */
|
174 | try {
|
175 | let g = this.connection(url);
|
176 | let data = await this._p_once(g);
|
177 | let res = data ? Object.keys(data).filter(k => k !== '_').sort().map(k => data[k]) : []; //See WORKAROUND-GUN-UNDERSCORE
|
178 | // .filter((obj) => (obj.signedby.includes(url))); // upper layers verify, which filters
|
179 | debuggun("p_rawlist found", ...utils.consolearr(res));
|
180 | return res;
|
181 | } catch(err) {
|
182 | // Will be logged by Transports
|
183 | throw(err);
|
184 | }
|
185 | }
|
186 |
|
187 | listmonitor(url, callback, {current=false}={}) {
|
188 | /*
|
189 | Setup a callback called whenever an item is added to a list, typically it would be called immediately after a p_rawlist to get any more items not returned by p_rawlist.
|
190 |
|
191 | url: string Identifier of list (as used by p_rawlist and "signedby" parameter of p_rawadd
|
192 | callback: function(obj) Callback for each new item added to the list
|
193 | obj is same format as p_rawlist or p_rawreverse
|
194 | current true if should send list of existing elements
|
195 | */
|
196 | let g = this.connection(url);
|
197 | if (!current) { // See WORKAROUND-GUN-CURRENT have to keep an extra copy to compare for which calls are new.
|
198 | g.once(data => {
|
199 | this.monitored = data ? Object.keys(data) : []; // Keep a copy - could actually just keep high water mark unless getting partial knowledge of state of array.
|
200 | g.map().on((v, k) => {
|
201 | if (!(this.monitored.includes(k)) && (k !== '_')) { //See WORKAROUND-GUN-UNDERSCORE
|
202 | this.monitored.push(k);
|
203 | callback(JSON.parse(v));
|
204 | }
|
205 | });
|
206 | });
|
207 | } else {
|
208 | g.map().on((v, k) => callback("set", k, JSON.parse(v)));
|
209 | }
|
210 | }
|
211 |
|
212 | // noinspection JSCheckFunctionSignatures
|
213 | async p_rawadd(url, sig) {
|
214 | /*
|
215 | Store a new list item, it should be stored so that it can be retrieved either by "signedby" (using p_rawlist) or
|
216 | by "url" (with p_rawreverse). The underlying transport does not need to guarantee the signature,
|
217 | an invalid item on a list should be rejected on higher layers.
|
218 |
|
219 | :param string url: String identifying list to post to
|
220 | :param Signature sig: Signature object containing at least:
|
221 | date - date of signing in ISO format,
|
222 | urls - array of urls for the object being signed
|
223 | signature - verifiable signature of date+urls
|
224 | signedby - urls of public key used for the signature
|
225 | :resolve undefined:
|
226 | */
|
227 | // noinspection JSUnresolvedVariable
|
228 | // Logged by Transports
|
229 | console.assert(url && sig.urls.length && sig.signature && sig.signedby.length, "TransportGUN.p_rawadd args", url, sig);
|
230 | this.connection(url)
|
231 | .set( canonicaljson.stringify( sig.preflight( Object.assign({}, sig))));
|
232 | }
|
233 |
|
234 | // noinspection JSCheckFunctionSignatures
|
235 | async p_newlisturls(cl) {
|
236 | let u = await this._p_newgun(cl);
|
237 | return [ u, u];
|
238 | }
|
239 |
|
240 | //=======KEY VALUE TABLES ========
|
241 |
|
242 | // noinspection JSMethodCanBeStatic
|
243 | async _p_newgun(pubkey) {
|
244 | if (pubkey.hasOwnProperty("keypair"))
|
245 | pubkey = pubkey.keypair.signingexport();
|
246 | // By this point pubkey should be an export of a public key of form xyz:abc where xyz
|
247 | // specifies the type of public key (NACL VERIFY being the only kind we expect currently)
|
248 | return `gun:/gun/${encodeURIComponent(pubkey)}`;
|
249 | }
|
250 | async p_newdatabase(pubkey) {
|
251 | /*
|
252 | Request a new database
|
253 | For GUN it doesnt actually create anything, just generates the URLs
|
254 | TODO-GUN simple version first - userid based on my keypair first, then switch to Gun's userid and its keypair
|
255 | Include gun/sea.js; user.create(<alias>,<passphrase>); user.auth(<alias>,<passphrase>); # See gun.eco/docs/Auth
|
256 |
|
257 | returns: {publicurl: "gun:/gun/<publickey>", privateurl: "gun:/gun/<publickey>">
|
258 | */
|
259 | let u = await this._p_newgun(pubkey);
|
260 | return {publicurl: u, privateurl: u};
|
261 | }
|
262 |
|
263 | async p_newtable(pubkey, table) {
|
264 | /*
|
265 | Request a new table
|
266 | For GUN it doesnt actually create anything, just generates the URLs
|
267 |
|
268 | returns: {publicurl: "gun:/gun/<publickey>/<table>", privateurl: "gun:/gun/<publickey>/<table>">
|
269 | */
|
270 | if (!pubkey) throw new errors.CodingError("p_newtable currently requires a pubkey");
|
271 | let database = await this.p_newdatabase(pubkey);
|
272 | // If have use cases without a database, then call p_newdatabase first
|
273 | return { privateurl: `${database.privateurl}/${table}`, publicurl: `${database.publicurl}/${table}`} // No action required to create it
|
274 | }
|
275 |
|
276 | async p_set(url, keyvalues, value) { // url = yjs:/yjs/database/table
|
277 | /*
|
278 | Set key values
|
279 | keyvalues: string (key) in which case value should be set there OR
|
280 | object in which case value is ignored
|
281 | */
|
282 | let table = this.connection(url);
|
283 | if (typeof keyvalues === "string") {
|
284 | table.path(keyvalues).put(canonicaljson.stringify(value));
|
285 | } else {
|
286 | // Store all key-value pairs without destroying any other key/value pairs previously set
|
287 | console.assert(!Array.isArray(keyvalues), "TransportGUN - shouldnt be passsing an array as the keyvalues");
|
288 | table.put(
|
289 | Object.keys(keyvalues).reduce(
|
290 | function(previous, key) { previous[key] = canonicaljson.stringify(keyvalues[key]); return previous; },
|
291 | {}
|
292 | ))
|
293 | }
|
294 | }
|
295 |
|
296 | async p_get(url, keys) {
|
297 | let table = this.connection(url);
|
298 | if (Array.isArray(keys)) {
|
299 | throw new errors.ToBeImplementedError("p_get(url, [keys]) isn't supported - because of ambiguity better to explicitly loop on set of keys or use getall and filter");
|
300 | /*
|
301 | return keys.reduce(function(previous, key) {
|
302 | let val = table.get(key);
|
303 | previous[key] = typeof val === "string" ? JSON.parse(val) : val; // Handle undefined
|
304 | return previous;
|
305 | }, {});
|
306 | */
|
307 | } else {
|
308 | let val = await this._p_once(table.get(keys)); // Resolves to value
|
309 | return typeof val === "string" ? JSON.parse(val) : val; // This looks like it is sync (see same code on p_get and p_rawfetch)
|
310 | }
|
311 | }
|
312 |
|
313 | async p_delete(url, keys) {
|
314 | let table = this.connection(url);
|
315 | if (typeof keys === "string") {
|
316 | table.path(keys).put(null);
|
317 | } else {
|
318 | keys.map((key) => table.path(key).put(null)); // This looks like it is sync
|
319 | }
|
320 | }
|
321 |
|
322 | //WORKAROUND-GUN-PROMISE suggest p_once as a good single addition
|
323 | //TODO-GUN expand this to workaround Gun weirdness with errors.
|
324 | _p_once(gun) { // Note in some cases (e.g. p_getall) this will resolve to a object, others a string/number (p_get)
|
325 | // TODO-GUN Temporarily added a 2000ms delay to workaround https://github.com/internetarchive/dweb-archive/issues/106 / https://github.com/amark/gun/issues/762
|
326 | return new Promise((resolve) => gun.once(resolve, {wait: 2000}));
|
327 | }
|
328 |
|
329 | async p_keys(url) {
|
330 | let res = await this._p_once(this.connection(url));
|
331 | return Object.keys(res)
|
332 | .filter(k=> (k !== '_') && (res[k] !== null)); //See WORKAROUND-GUN-UNDERSCORE and WORKAROUND-GUN-DELETE
|
333 | }
|
334 |
|
335 | async p_getall(url) {
|
336 | let res = await this._p_once(this.connection(url));
|
337 | return Object.keys(res)
|
338 | .filter(k=> (k !== '_') && res[k] !== null) //See WORKAROUND-GUN-UNDERSCORE and WORKAROUND-GUN-DELETE
|
339 | .reduce( function(previous, key) { previous[key] = JSON.parse(res[key]); return previous; }, {});
|
340 | }
|
341 |
|
342 | async monitor(url, callback, {current=false}={}) {
|
343 | /*
|
344 | Setup a callback called whenever an item is added to a list, typically it would be called immediately after a p_getall to get any more items not returned by p_getall.
|
345 | Stack: KVT()|KVT.p_new => KVT.monitor => (a: Transports.monitor => GUN.monitor)(b: dispatchEvent)
|
346 |
|
347 | url: string Identifier of list (as used by p_rawlist and "signedby" parameter of p_rawadd
|
348 | callback: function({type, key, value}) Callback for each new item added to the list (type = "set"|"delete")
|
349 | current Send existing items to the callback as well
|
350 | */
|
351 | let g = this.connection(url);
|
352 | if (!current) { // See WORKAROUND-GUN-CURRENT have to keep an extra copy to compare for which calls are new.
|
353 | g.once(data => {
|
354 | this.monitored = Object.assign({},data); // Make a copy of data (this.monitored = data won't work as just points at same structure)
|
355 | g.map().on((v, k) => {
|
356 | if ((v !== this.monitored[k]) && (k !== '_')) { //See WORKAROUND-GUN-UNDERSCORE
|
357 | this.monitored[k] = v;
|
358 | callback("set", k, JSON.parse(v));
|
359 | }
|
360 | });
|
361 | });
|
362 | } else {
|
363 | g.map().on((v, k) => callback("set", k, JSON.parse(v)));
|
364 | }
|
365 | }
|
366 |
|
367 | static async p_test() {
|
368 | debuggun("p_test");
|
369 | try {
|
370 | let t = this.setup0({}); //TODO-GUN when works with peers commented out, try passing peers: []
|
371 | await t.p_setup1(); // Not passing cb yet
|
372 | await t.p_setup2(); // Not passing cb yet - this one does nothing on GUN
|
373 | // noinspection JSIgnoredPromiseFromCall
|
374 | t.p_test_kvt("gun:/gun/NACL");
|
375 | //t.p_test_list("gun:/gun/NACL"); //TODO test_list needs fixing to not create a dependency on Signature
|
376 | } catch(err) {
|
377 | console.warn("Exception thrown in TransportGUN.test:", err.message);
|
378 | throw err;
|
379 | }
|
380 | }
|
381 |
|
382 | // noinspection JSUnusedGlobalSymbols
|
383 | static async demo_bugs() {
|
384 | let gun = new Gun();
|
385 | gun.get('foo').get('bar').put('baz');
|
386 | console.log("Expect {bar: 'baz'} but get {_:..., bar: 'baz'}");
|
387 | gun.get('foo').once(data => console.log(data));
|
388 | gun.get('zip').get('bar').set('alice');
|
389 | console.log("Expect {12345: 'alice'} but get {_:..., 12345: 'alice'}");
|
390 | gun.get('foo').once(data => console.log(data));
|
391 | // Returns extra "_" field
|
392 | }
|
393 | }
|
394 | Transports._transportclasses["GUN"] = TransportGUN;
|
395 | // Defines global.Gun
|
396 | TransportGUN.requires = TransportGUN.scripts = ['gun/gun.js', 'gun/lib/path.js', 'gun/nts', 'gun/lib/wire', 'gun/lib/multicast', 'gun/lib/radix.js',
|
397 | 'gun/lib/radisk.js', 'gun/lib/store.js', 'gun/lib/rindexed.js'];
|
398 | exports = module.exports = TransportGUN;
|