#!/bin/bash

# Sleep for a given grace period before counting failures
sleep ${GRACE_PERIOD:-0}

while sleep ${INTERVAL:-2}; do
  export consul_failed=0
  export consul_container=$(docker ps --format "{{ .Image }} {{ .Names }}" | grep consul | awk '{ print $2 }')
  if $(curl -sf $CONSUL_URL/v1/status/leader); then
    export consul_failed=0
  else
    export consul_failed=$(($consul_failed + 1))
  fi

  if [[ "$consul_failed" -gt "2" ]]; then
    echo reaping $consul_container
    export consul_failed=0
    docker -H ${DOCKER_HOST:-unix:///var/run/docker.sock} kill $consul_container || true
  fi

  passing=$(curl -s $CONSUL_URL/v1/health/node/$HOSTNAME | jq -r '.[] | select(.Status=="passing") | .ServiceID' | awk -F : '{ print $2":"$3 }' | sort -u)
  failing=$(curl -s $CONSUL_URL/v1/health/node/$HOSTNAME | jq -r '.[] | select(.Status=="warning" or .Status=="critical") | .ServiceID' | awk -F : '{ print $2":"$3 }' | sort -u)
  passing_not_failing=$(comm -23 <(echo $passing | tr ' ' '\n' | sort) <(echo $failing | tr ' ' '\n' | sort))

  # Reset fail count for each passing service

  for service in $passing_not_failing; do
    name=$(echo $service | tr -d '[:punct:]')
    export ${name}_container=$(echo $service | awk -F : '{ print $1 }')
    export container=${name}_container
    export failed=${name}_failed
    if [[ "${failed}" -gt "1" ]]; then
      curl -X POST -H 'Content-type: application/json' \
        --data "{ \
          \"attachments\": [{ \
            \"text\": \":thumbsup: Service $service on host $HOSTNAME is now healthy\", \
            \"fallback\": \":thumbsup: Service $service on host $HOSTNAME is now healthy\", \
            \"color\": \"good\",
            \"author_name\": \"${!container}.$HOSTNAME.$DOMAIN\",
            \"author_link\": \"consul.$DOMAIN/ui/#/dc1/nodes/$HOSTNAME\"
          }], \
          \"channel\": \"#$SLACK_CHANNEL\" \
        }" \
        $SLACK_HOOK
    fi
    export ${name}_failed=0
  done

  # Increment fail count for each failed service

  for service in $failing; do
    export name=$(echo $service | tr -d '[:punct:]')
    export ${name}_container=$(echo $service | awk -F : '{ print $1 }')
    export container=${name}_container
    export failed=${name}_failed
    export ${name}_failed=$((${!failed:-0} + 1))
    echo "${!container} failed ${!failed} times"

    if [[ "${!failed}" -gt "1" ]]; then
      curl -X POST -H 'Content-type: application/json' \
          --data "{ \
            \"attachments\": [{ \
              \"text\": \":scream: Service $service on host $HOSTNAME has failed ${!failed} consecutive checks\", \
              \"fallback\": \":scream: Service $service on host $HOSTNAME has failed ${!failed} consecutive checks\", \
              \"color\": \"danger\",
              \"author_name\": \"${!container}.$HOSTNAME.$DOMAIN\",
              \"author_link\": \"consul.$DOMAIN/ui/#/dc1/nodes/$HOSTNAME\"
            }], \
            \"channel\": \"#$SLACK_CHANNEL\" \
          }" \
          $SLACK_HOOK
    fi

    # Restart container if fail count exceeds max

    if [[ "${!failed}" -gt ${MAX_FAILURES:-3} ]]; then

      echo reaping ${!container}
      docker -H ${DOCKER_HOST:-unix:///var/run/docker.sock} kill ${!container} || true

      curl -X POST -H 'Content-type: application/json' \
        --data "{ \
          \"attachments\": [{ \
            \"text\": \":skull: Container ${!container} on host $HOSTNAME has been reaped\", \
            \"fallback\": \":skull: Container ${!container} on host $HOSTNAME has been reaped\", \
            \"color\": \"warning\",
            \"author_name\": \"${!container}.$HOSTNAME.$DOMAIN\",
            \"author_link\": \"consul.$DOMAIN/ui/#/dc1/nodes/$HOSTNAME\"
          }], \
          \"channel\": \"#$SLACK_CHANNEL\" \
        }" \
        $SLACK_HOOK

      # reset fail counts for all services with same container

      for other_service in $passing_not_failing; do
        export other_name=$(echo $other_service | tr -d '[:punct:]')
        export other_container=$(echo $other_service | awk -F : '{ print $1 }')
        if [[ "${other_container}" == "${!container}" ]]; then
          export ${other_name}_failed=0
        fi
      done

      for other_service in $failing; do
        export other_name=$(echo $other_service | tr -d '[:punct:]')
        export other_container=$(echo $other_service | awk -F : '{ print $1 }')
        if [[ "${other_container}" == "${!container}" ]]; then
          export ${other_name}_failed=0
        fi
      done
    fi
  done
done
