# EXPRESSER DOWNLOADER # -------------------------------------------------------------------------- # Handles external downloads. # class Downloader events = require "./events.coffee" fs = require "fs" http = require "http" https = require "https" lodash = require "lodash" logger = require "./logger.coffee" moment = require "moment" path = require "path" settings = require "./settings.coffee" url = require "url" # The download queue and simultaneous count. queue = [] downloading = [] # CONSTRUCTOR AND INIT # -------------------------------------------------------------------------- # Downloader constructor. constructor: -> @setEvents() if settings.events.enabled # Bind event listeners. setEvents: => events.on "downloader.download", @download # METHODS # -------------------------------------------------------------------------- # Download an external file and save it to the specified location. The `callback` # has the signature (error, data). Returns the downloader object which is added # to the `queue`, which has the download properties and a `stop` helper to force # stopping it. Returns false on error or duplicate. # Tip: if you want to get the downloaded data without having to read the target file # you can get the downloaded contents via the `options.downloadedData`. # @param [String] remoteUrl The URL of the remote file to be downloaded. # @param [String] saveTo The full local path and destination filename. # @param [Object] options Optional, object with request options, for example auth. # @param [Method] callback Optional, a function (err, result) to be called when download has finished. # @return [Object] Returns the download job having timestamp, remoteUrl, saveTo, options, callback and stop helper. download: (remoteUrl, saveTo, options, callback) => if not remoteUrl? logger.warn "Downloader.download", "Aborted, remoteUrl is not defined." return # Check options and callback. if not callback? and lodash.isFunction options callback = options options = null now = new Date().getTime() # Create the download object. downloadObj = {timestamp: now, remoteUrl: remoteUrl, saveTo: saveTo, options: options, callback: callback} # Prevent duplicates? if settings.downloader.preventDuplicates existing = lodash.filter downloading, {remoteUrl: remoteUrl, saveTo: saveTo} # If downloading the same file and to the same location, abort download. if existing.length > 0 existing = existing[0] if existing.saveTo is saveTo logger.warn "Downloader.download", "Aborted, already downloading.", remoteUrl, saveTo err = {message: "Download aborted: same file is already downloading.", duplicate: true} callback(err, downloadObj) if callback? return false # Create a `stop` method to force stop the download by setting the `stopFlag`. # Accepts a `keep` boolean, if true the already downloaded data will be kept on forced stop. stopHelper = (keep) -> @stopFlag = (if keep then 1 else 2) # Update download object with stop helper and add to queue. downloadObj.stop = stopHelper queue.push downloadObj # Start download immediatelly if not exceeding the `maxSimultaneous` setting. next() if downloading.length < settings.downloader.maxSimultaneous return downloadObj # INTERNAL IMPLEMENTATION # -------------------------------------------------------------------------- # Helper to remove a download from the `downloading` list. removeDownloading = (obj) -> filter = {timestamp: obj.timestamp, remoteUrl: obj.remoteUrl, saveTo: obj.saveTo} downloading = lodash.reject downloading, filter # Helper function to proccess download errors. downloadError = (err, obj) -> logger.debug "Downloader.downloadError", err, obj removeDownloading obj next() obj.callback(err, obj) if obj.callback? # Helper function to parse the URL and get its options. parseUrlOptions = (obj, options) -> if obj.redirectUrl? and obj.redirectUrl isnt "" urlInfo = url.parse obj.redirectUrl else urlInfo = url.parse obj.remoteUrl # Set URL options. options = host: urlInfo.hostname hostname: urlInfo.hostname port: urlInfo.port path: urlInfo.path # Check for credentials on the URL. if urlInfo.auth? and urlInfo.auth isnt "" options.auth = urlInfo.auth return options # Helper function to start a download request. reqStart = (obj, options) -> if obj.remoteUrl.indexOf("https") is 0 options.port = 443 if not options.port? httpHandler = https else httpHandler = http # Start the request. req = httpHandler.get options, (response) => # Downloaded contents will be appended also to the `downloadedData` # property of the options object. obj.downloadedData = "" # Set the estination temp file. saveToTemp = obj.saveTo + settings.downloader.tempExtension # If status is 301 or 302, redirect to the specified location and stop the current request. if response.statusCode is 301 or response.statusCode is 302 obj.redirectUrl = response.headers.location options = lodash.assign options, parseUrlOptions obj req.end() reqStart obj, options # If status is not 200 or 304, it means something went wrong so do not proceed # with the download. Otherwise proceed and listen to the `data` and `end` events. else if response.statusCode isnt 200 and response.statusCode isnt 304 err = {code: response.statusCode, message: "Server returned an unexpected status code: #{response.statusCode}"} downloadError err, obj else # Create the file stream with a .download extension. This will be renamed after the # download has finished and the file is totally written. fileWriter = fs.createWriteStream saveToTemp, {"flags": "w+"} # Helper called response gets new data. The data will also be # appended to `options.data` property. onData = (data) -> if obj.stopFlag req.end() onEnd() else fileWriter.write data obj.downloadedData += data # Helper called when response ends. onEnd = -> response.removeListener "data", onData fileWriter.addListener "close", -> # Check if temp file exists. if fs.existsSync? tempExists = fs.existsSync saveToTemp else tempExists = path.existsSync saveToTemp # If temp download file can't be found, set error message. # If `stopFlag` is 2 means download was stopped and should not keep partial data. if not tempExists err = {message:"Can't find downloaded file: #{saveToTemp}"} else fs.unlinkSync saveToTemp if obj.stopFlag is 2 # Check if destination file already exists. if fs.existsSync? fileExists = fs.existsSync obj.saveTo else fileExists = path.existsSync obj.saveTo # Only proceed with renaming if `stopFlag` wasn't set and destionation is valid. if not obj.stopFlag? or obj.stopFlag < 1 fs.unlinkSync obj.saveTo if fileExists fs.renameSync saveToTemp, obj.saveTo if tempExists # Remove from `downloading` list and proceed with the callback. removeDownloading obj obj.callback(err, obj) if obj.callback? logger.debug "Downloader.next", "End", obj.remoteUrl, obj.saveTo fileWriter.end() fileWriter.destroySoon() next() # Attachd response listeners. response.addListener "data", onData response.addListener "end", onEnd # Unhandled error, call the downloadError helper. req.on "error", (err) => downloadError err, obj # Process next download. next = -> return if queue.length < 0 # Get first download from queue. obj = queue.shift() # Check if download is valid. if not obj? logger.debug "Downloader.next", "Skip", "Downloader object is invalid." return else logger.debug "Downloader.next", obj # Add to downloading array. downloading.push obj if settings.downloader.headers? and settings.downloader.headers isnt "" headers = settings.web.downloaderHeaders else headers = null # Set default options. options = headers: headers rejectUnauthorized: settings.downloader.rejectUnauthorized # Extend options. options = lodash.assign options, obj.options, parseUrlOptions(obj) # Start download if obj.stopFlag? and obj.stopFlag > 0 logger.debug "Downloader.next", "Skip, 'stopFlag' is #{obj.stopFlag}.", obj removeDownloading obj next() else reqStart obj, options # Singleton implementation # -------------------------------------------------------------------------- Downloader.getInstance = -> @instance = new Downloader() if not @instance? return @instance module.exports = exports = Downloader.getInstance()