1 | const Transport = require('./Transport'); // Base class for TransportXyz
|
2 | const Transports = require('./Transports'); // Manage all Transports that are loaded
|
3 | const httptools = require('./httptools'); // Expose some of the httptools so that IPFS can use it as a backup
|
4 | const Url = require('url');
|
5 | const stream = require('readable-stream');
|
6 | const debug = require('debug')('dweb-transports:http');
|
7 | //TODO-SPLIT pull /arc out of here, then dont need by default to hearbeat to dweb.me
|
8 |
|
9 | defaulthttpoptions = {
|
10 | urlbase: 'https://dweb.me',
|
11 | heartbeat: { delay: 30000 } // By default check twice a minute
|
12 | };
|
13 |
|
14 | class TransportHTTP extends Transport {
|
15 | /* Subclass of Transport for handling HTTP - see API.md for docs
|
16 |
|
17 | options {
|
18 | urlbase: e.g. https://dweb.me Where to go for URLS like /arc/...
|
19 | heartbeat: {
|
20 | delay // Time in milliseconds between checks - 30000 might be appropriate - if missing it wont do a heartbeat
|
21 | statusCB // Callback cb(transport) when status changes
|
22 | }
|
23 | }
|
24 | */
|
25 |
|
26 | constructor(options) {
|
27 | super(options); // These are now options.http
|
28 | this.options = options;
|
29 | this.urlbase = options.urlbase; // e.g. https://dweb.me
|
30 | this.supportURLs = ['http','https'];
|
31 | this.supportFunctions = ['fetch'];
|
32 | this.supportFeatures = ['noCache'];
|
33 | if (typeof window === "undefined") {
|
34 | // running in node, can support createReadStream, (browser can't - see createReadStream below)
|
35 | this.supportFunctions.push("createReadStream");
|
36 | }
|
37 | // noinspection JSUnusedGlobalSymbols
|
38 | this.supportFeatures = ['fetch.range', 'noCache'];
|
39 | this.name = "HTTP"; // For console log etc
|
40 | this.status = Transport.STATUS_LOADED;
|
41 | }
|
42 |
|
43 | static setup0(options) {
|
44 | let combinedoptions = Transport.mergeoptions(defaulthttpoptions, options.http);
|
45 | try {
|
46 | let t = new TransportHTTP(combinedoptions);
|
47 | Transports.addtransport(t);
|
48 | return t;
|
49 | } catch (err) {
|
50 | debug("ERROR: HTTP unable to setup0", err.message);
|
51 | throw err;
|
52 | }
|
53 | }
|
54 |
|
55 | p_setup1(statusCB) {
|
56 | return new Promise((resolve, unusedReject) => {
|
57 | this.status = Transport.STATUS_STARTING;
|
58 | if (statusCB) statusCB(this);
|
59 | this.updateStatus((unusedErr, unusedRes) => {
|
60 | if (statusCB) statusCB(this);
|
61 | this.startHeartbeat(this.options.heartbeat);
|
62 | resolve(this); // Note always resolve even if error from p_status as have set status to failed
|
63 | });
|
64 | })
|
65 | }
|
66 |
|
67 | async p_status(cb) { //TODO-API
|
68 | /*
|
69 | Return (via cb or promise) a numeric code for the status of a transport.
|
70 | */
|
71 | if (cb) { try { this.updateStatus(cb) } catch(err) { cb(err)}} else { return new Promise((resolve, reject) => { try { this.updateStatus((err, res) => { if (err) {reject(err)} else {resolve(res)} })} catch(err) {reject(err)}})} // Promisify pattern v2f
|
72 | }
|
73 | updateStatus(cb) { //TODO-API
|
74 | this.updateInfo((err, res) => {
|
75 | if (err) {
|
76 | debug("Error status call to info failed %s", err.message);
|
77 | this.status = Transport.STATUS_FAILED;
|
78 | cb(null, this.status); // DOnt pass error up, the status indicates the error
|
79 | } else {
|
80 | this.info = res; // Save result
|
81 | this.status = Transport.STATUS_CONNECTED;
|
82 | cb(null, this.status);
|
83 | }
|
84 | });
|
85 | }
|
86 |
|
87 | startHeartbeat({delay=undefined, statusCB=undefined}) {
|
88 | if (delay) {
|
89 | debug("%s Starting Heartbeat", this.name)
|
90 | this.heartbeatTimer = setInterval(() => {
|
91 | this.updateStatus((err, res)=>{ // Pings server and sets status
|
92 | if (statusCB) statusCB(this); // repeatedly call callback if supplies
|
93 | }, (unusedErr, unusedRes)=>{}); // Dont wait for status to complete
|
94 | }, delay);
|
95 | }
|
96 | }
|
97 | stopHeartbeat() {
|
98 | if (this.heartbeatTimer) {
|
99 | debug("stopping heartbeat");
|
100 | clearInterval(this.heartbeatTimer);}
|
101 | }
|
102 | stop(refreshstatus, cb) {
|
103 | this.stopHeartbeat();
|
104 | this.status = Transport.STATUS_FAILED;
|
105 | if (refreshstatus) { refreshstatus(this); }
|
106 | cb(null, this);
|
107 | }
|
108 |
|
109 | validFor(url, func, opts) {
|
110 | // Overrides Transport.prototype.validFor because HTTP's connection test is only really for dweb.me
|
111 | // in particular this allows urls like https://be-api.us.archive.org
|
112 | return (this.connected() || (url.protocol.startsWith("http") && ! url.href.startsWith(this.urlbase))) && this.supports(url, func, opts);
|
113 | }
|
114 | async p_rawfetch(url, opts={}) {
|
115 | /*
|
116 | Fetch from underlying transport,
|
117 | Fetch is used both for contenthash requests and table as when passed to SmartDict.p_fetch may not know what we have
|
118 | url: Of resource - which is turned into the HTTP url in p_httpfetch
|
119 | opts: {start, end, retries, noCache} see p_GET for documentation
|
120 | throws: TransportError if fails
|
121 | */
|
122 | return await httptools.p_GET(url, opts);
|
123 | }
|
124 |
|
125 | // ============================== Stream support
|
126 |
|
127 | async p_f_createReadStream(url, {wanturl=false}={}) {
|
128 | /*
|
129 | Fetch bytes progressively, using a node.js readable stream, based on a url of the form:
|
130 | No assumption is made about the data in terms of size or structure.
|
131 |
|
132 | This is the initialisation step, which returns a function suitable for <VIDEO>
|
133 |
|
134 | Returns a new Promise that resolves to function for a node.js readable stream.
|
135 |
|
136 | Node.js readable stream docs: https://nodejs.org/api/stream.html#stream_readable_streams
|
137 |
|
138 | :param string url: URL of object being retrieved of form magnet:xyzabc/path/to/file (Where xyzabc is the typical magnet uri contents)
|
139 | :param boolean wanturl True if want the URL of the stream (for service workers)
|
140 | :resolves to: f({start, end}) => stream (The readable stream.)
|
141 | :throws: TransportError if url invalid - note this happens immediately, not as a catch in the promise
|
142 | */
|
143 | //Logged by Transports
|
144 | //debug("p_f_createreadstream %s", Url.parse(url).href);
|
145 | try {
|
146 | let self = this;
|
147 | if (wanturl) {
|
148 | return url;
|
149 | } else {
|
150 | return function (opts) { return self.createReadStream(url, opts); };
|
151 | }
|
152 | } catch(err) {
|
153 | //Logged by Transports
|
154 | //console.warn(`p_f_createReadStream failed on ${Url.parse(url).href} ${err.message}`);
|
155 | throw(err);
|
156 | }
|
157 | }
|
158 |
|
159 | createReadStream(url, opts) {
|
160 | /*
|
161 | The function, encapsulated and inside another function by p_f_createReadStream (see docs)
|
162 | NOTE THIS DOESNT WONT WORK FOR <VIDEO> tags, but shouldnt be using it there anyway - reports stream.on an filestream.pipe aren't functions
|
163 |
|
164 | :param file: Webtorrent "file" as returned by webtorrentfindfile
|
165 | :param opts: { start: byte to start from; end: optional end byte }
|
166 | :returns stream: The readable stream - it is returned immediately, though won't be sending data until the http completes
|
167 | */
|
168 | // This breaks in browsers ... as 's' doesn't have .pipe but has .pipeTo and .pipeThrough neither of which work with stream.PassThrough
|
169 | // TODO See https://github.com/nodejs/readable-stream/issues/406 in case its fixed in which case enable createReadStream in constructor above.
|
170 | debug("createreadstream %s %o", Url.parse(url).href, opts);
|
171 | let through;
|
172 | through = new stream.PassThrough();
|
173 | httptools.p_GET(url, Object.assign({wantstream: true}, opts))
|
174 | .then(s => s.pipe(through))
|
175 | // Note any .catch is happening AFTER through returned
|
176 | .catch(err => {
|
177 | console.warn(this.name, "createReadStream caught error", err.message);
|
178 | if (typeof through.destroy === 'function') {
|
179 | through.destroy(err); // Will emit error & close and free up resources
|
180 | // caller MUST implimit through.on('error', err=>) or will generate uncaught error message
|
181 | } else {
|
182 | through.emit('error', err);
|
183 | }
|
184 | });
|
185 | return through; // Returns "through" synchronously, before the pipe is setup
|
186 | }
|
187 |
|
188 | async p_createReadStream(url, opts) {
|
189 | /*
|
190 | The function, encapsulated and inside another function by p_f_createReadStream (see docs)
|
191 | NOTE THIS PROBABLY WONT WORK FOR <VIDEO> tags, but shouldnt be using it there anyway
|
192 |
|
193 | :param file: Webtorrent "file" as returned by webtorrentfindfile
|
194 | :param opts: { start: byte to start from; end: optional end byte }
|
195 | :resolves to stream: The readable stream.
|
196 | */
|
197 | debug("createreadstream %s %o", Url.parse(url).href, opts);
|
198 | try {
|
199 | return await httptools.p_GET(url, Object.assign({wantstream: true}, opts));
|
200 | } catch(err) {
|
201 | console.warn(this.name, "caught error", err);
|
202 | throw err;
|
203 | }
|
204 | }
|
205 |
|
206 | async p_info() { //TODO-API
|
207 | /*
|
208 | Return (via cb or promise) a numeric code for the status of a transport.
|
209 | */
|
210 | return new Promise((resolve, reject) => { try { this.updateInfo((err, res) => { if (err) {reject(err)} else {resolve(res)} })} catch(err) {reject(err)}}) // Promisify pattern v2b (no CB)
|
211 | }
|
212 |
|
213 | updateInfo(cb) {
|
214 | httptools.p_GET(`${this.urlbase}/info`, {retries: 1}, cb); // Try info, but dont retry (usually heartbeat will reconnect)
|
215 | }
|
216 |
|
217 | static async p_test(opts={}) {
|
218 | {console.log("TransportHTTP.test")}
|
219 | try {
|
220 | let transport = await this.p_setup(opts);
|
221 | console.log("HTTP connected");
|
222 | let res = await transport.p_info();
|
223 | console.log("TransportHTTP info=",res);
|
224 | res = await transport.p_status();
|
225 | console.assert(res === Transport.STATUS_CONNECTED);
|
226 | await transport.p_test_kvt("NACL%20VERIFY");
|
227 | } catch(err) {
|
228 | console.log("Exception thrown in TransportHTTP.test:", err.message);
|
229 | throw err;
|
230 | }
|
231 | }
|
232 |
|
233 | static async test() {
|
234 | return this;
|
235 | }
|
236 |
|
237 | }
|
238 | Transports._transportclasses["HTTP"] = TransportHTTP;
|
239 | TransportHTTP.requires = TransportHTTP.scripts = []; // Nothing to load
|
240 | exports = module.exports = TransportHTTP;
|
241 |
|