# All this rigmarole only to concurrently make some http get requests
# :(

events = require 'events'
util = require 'util'

Q = require 'q'
RateLimiter = require('limiter').RateLimiter
request = require 'request'

u = require './utils'
meta = require '../package.json'

# Example:
#
# class MyError extends Error
#   constructor: (msg) -> superError MyError, this, msg
#
superError = (klass, errObj, msg) ->
  errObj.name = klass.name
  errObj.message = msg
  Error.captureStackTrace errObj, klass

class CrawlerError extends Error
  constructor: (msg) -> superError CrawlerError, this, msg

class Dup extends CrawlerError
  constructor: (msg) -> superError Dup, this, msg

class InvalidJSON extends CrawlerError
  constructor: (msg) -> superError InvalidJSON, this, msg

class InvalidItem extends CrawlerError
  constructor: (msg) -> superError InvalidItem, this, msg

class NoItem extends CrawlerError
  constructor: (msg) -> superError NoItem, this, msg

class PolloptInIteration extends CrawlerError
  constructor: (msg) -> superError PolloptInIteration, this, msg

class NoConnect extends CrawlerError
  constructor: (msg) -> superError NoConnect, this, msg

makeError = (klass, iter_id, id, msg) ->
  err = new klass msg
  err.iter_id = iter_id
  err.id = id
  err

class Stat

  constructor: (@event)->
    @iter_id = 0
    @_history = {}
    @bytes = 0
    @_iterations = []

  # id -- item id
  history: (iter_id, id, err, iter_id_next = null) ->
    o = @_history[iter_id] ||= {}
    o[id] ||= []
    last = o[id][o[id].length-1]
    o[id].pop() if last?.iter_id_next == 'http get'

    info = {}
    info.err = err if err
    info.iter_id_next = iter_id_next.toString() if iter_id_next
    o[id].push info
    @event.emit 'herr', err if info.err

  iter_id_next: ->
    ++@iter_id

  has_been: (id) ->
    id = id.toString()
    for key,val of @_history
      return true if id of val
    false

  succinct: ->
    ids = {}
    for ikey,ival of @_history
      for id,val of ival
        ids[id] ||= []
        ids[id].push val

    r =
      bytes: @bytes
      items: 0
      neterr: 0
      invalid: 0
      dup: 0
      uniq: (Object.keys ids).length

    errclass2desc =
      'Dup': 'dup'
      'InvalidJSON': 'invalid'
      'InvalidItem': 'invalid'
      'NoItem': 'neterr'
      'PolloptInIteration': 'invalid'
      'NoConnect': 'neterr'

    for id,val of ids
      for items in val
        for item in items
          if !item.err
            r.items++
          else
            r[errclass2desc[item.err.constructor.name]]++

    r

  iteration_begin: (iter_id) ->
    idx = @_iterations.indexOf iter_id
    if idx == -1
      @_iterations.push iter_id
      @event.emit 'iteration:begin', iter_id

  iteration_end: (iter_id) ->
    idx = @_iterations.indexOf iter_id
    unless idx == -1
      @_iterations.splice idx, 1
      @event.emit 'iteration:end', iter_id
      @event.emit 'finish' if @_iterations.length == 0

class Crawler2

  @ItemValidate = (json) ->
    return false unless json
    return false if !json.id
    return false if !json.type?.toString().match /^(story|comment|poll|pollopt)$/
    return false if json.kids && !util.isArray json.kids

    return false if json.type == 'poll' && !json.parts
    return false if json.type == 'poll' && !util.isArray json.parts
    return false if json.type == 'poll' && json.parts.length == 0

    true

  @idValidate = (id, deferred) ->
    unless u.isNum id
      deferred.reject new Error "invalid id `#{id}`"
      return false
    true

  @iterIdValidate = (iter_id, deferred) ->
    unless u.isStr(iter_id) || u.isNum(iter_id)
      deferred.reject new Error "invalid iter_id `#{iter_id}`"
      return false
    true

  @CONN_PER_SEC = 20

  constructor: (@conn_per_sec = Crawler2.CONN_PER_SEC) ->
    @url_pattern = 'https://hacker-news.firebaseio.com/v0/item/%s.json'
    @look4kids = true
    @logger = console.error
    @event = new events.EventEmitter()
    @stat = new Stat @event
    @limiter = new RateLimiter(@conn_per_sec, 'sec')

    @event.on 'items', (iter_id, idarr) =>
      @iteration iter_id, idarr

    @event.on 'poll', (iter_id, item) =>
      @poll iter_id, item

    @http_headers = {
      'User-Agent': [meta.name, meta.version].join '/'
    }

  # Emit 'data' event if an item was fetched. Emit 'poll' event if the
  # fetched item was a poll. Emit 'item' event if the fetched item had
  # kids.
  iteration: (iter_id, idarr) ->
    throw new Error "array of IDs is expected" unless util.isArray idarr

    for id,idx in idarr
      @stat.iteration_begin iter_id if idx == 0

      do (idx) =>
        @fetch_item(iter_id, id)
        .then (r) =>
          if r.type == 'pollopt'
            throw makeError PolloptInIteration, iter_id, r.id, "must be processed by Crawler2#poll()"

          if r.type == 'poll'
            iter_id_next = @stat.iter_id_next() + '.poll'
            @stat.history iter_id, r.id, null, iter_id_next
            @event.emit 'poll', iter_id_next, r
          else
            @event.emit 'data', iter_id, JSON.stringify r
            @stat.history iter_id, r.id, null, null unless @iteration_4kids iter_id, r

        .catch (err) =>
          if err instanceof CrawlerError
            @stat.history iter_id, err.id, err, null
          else
            throw err
        .fin =>
          @stat.iteration_end iter_id if idx == idarr.length-1
        .done()

  iteration_4kids: (iter_id, item) ->
    if item.kids && @look4kids
      iter_id_next = @stat.iter_id_next()
      @stat.history iter_id, item.id, null, iter_id_next
      @event.emit 'items', iter_id_next, item.kids

  # Emit a 'data' msg if a poll was collected w/o errors.
  #
  # If there are errors in collecting of any poll parts, record the fact
  # in the @stat for current iteration.
  #
  # Side effects: modifies @stat
  # Return: nothing
  poll: (iter_id, item) ->
    @iteration_4kids iter_id, item

    # include poll item itself before its additional parts
    parts = [JSON.stringify item]

    promises = []
    for id,idx in item.parts
      @stat.iteration_begin iter_id if idx == 0

      do (idx) =>
        p = @fetch_item(iter_id, id)
        promises.push p

        p.then (r) =>
          @stat.history iter_id, r.id, null, null
          parts.push JSON.stringify r
        .catch (err) =>
          # current pollopt is invalid
          if err instanceof CrawlerError
            @stat.history iter_id, err.id, err, null
          else
            throw err
        .fin =>
          @stat.iteration_end iter_id if idx == item.parts.length-1
        .done()

    # wait for all promises to finish
    Q.all(promises).then =>
      @event.emit 'data', iter_id, parts.join "\n"

  # Side effects: modifies @stat, @stat.bytes
  # Return: a promise w/ a response body
  http_get: (iter_id, id) ->
    log = (msg) => @logger "ii=#{iter_id}/id=#{id} #{msg}"
    deferred = Q.defer()

    opt =
      url: util.format @url_pattern, id
      headers: @http_headers
    cur_req = request.get opt, (err, res, body) =>
      if err
        deferred.reject makeError NoConnect, iter_id, id, err.message
        return deferred.promise

      if res.statusCode == 200
        @stat.bytes += Buffer.byteLength body
        log "HTTP 200"
        deferred.resolve body
      else
        deferred.reject makeError NoItem, iter_id, id, "HTTP #{res.statusCode}"

    cur_req.on 'request', =>
      # check in history, mark as dup; ignore polls iterations
      if @stat.has_been id
        unless iter_id.toString().match /\.poll$/
          deferred.reject makeError Dup, iter_id, id, "dup!"
          cur_req.abort()
      else
        log "HTTP GET"
        @stat.history iter_id, id, null, 'http get'

    deferred.promise

  # Fetch id via http, parse it & set the result in `deferred`.
  #
  # Side effects: modifies @stat
  # Return: nothing
  fetch_item_bare: (iter_id, id, deferred) ->
    return unless Crawler2.idValidate id, deferred
    return unless Crawler2.iterIdValidate iter_id, deferred

    @http_get iter_id, id
    .then (item) ->
      try
        item = JSON.parse item
      catch e
        deferred.reject makeError InvalidJSON, iter_id, id, "invalid JSON"
        return

      if Crawler2.ItemValidate item
        deferred.resolve item
      else
        deferred.reject makeError InvalidItem, iter_id, id, "invalid data in the item"
    .catch (err) ->
      deferred.reject err
    .done()

  # A wrapper for #fetch_item_bare(). Limits executions of
  # #fetch_item_bare() by @conn_per_sec.
  #
  # Side effects: modifies @stat
  # Return: a promise w/ the result from #fetch_item_bare()
  fetch_item: (iter_id, id) ->
    deferred = Q.defer()
    @limiter.removeTokens 1, (lim_err, remainingRequests) =>
      @fetch_item_bare iter_id, id, deferred
    deferred.promise


module.exports = Crawler2