description "monitor cfk for missing dns entries"

start on starting weave
stop on stopping weave

script
  set +x
{% for key, value in services.iteritems() %}
  {{ key | regex_replace('[.-]', '_') | upper }}_NO_DNS=0
{% endfor %}

  while true; do
    sleep 30

    eval $(cfk-env)

    # forget unreachable peers
    weave status ipam | grep unreachable | awk '{ print $1 }' | tr '(' ' ' | awk '{ print $1 }' | xargs -l weave forget || true

    # get machine name
    NAME=$(weave status | grep Name | awk '{ print $2 }' | cut -d '(' -f 1)

    # collect dns entries
    WEAVE_DNS=$(weave status dns)

    # write peers to file
    (weave status targets | awk '{ printf $1" "; system("curl -s -f "$1"/cfk-role"); print ""}' \
      && echo "${HOST_IP} ${CFK_ROLE}") \
      > /var/run/weave/peers

    # prompt consul to sync peers
    consul_id=$(docker ps --format "{{ "{{" }} .Image {{ "}}" }} {{ "{{" }} .ID {{ "}}" }}" | grep consul | awk '{ print $2 }')
    if [ ! -z "$consul_id" ]; then
      # join previously unknown masters
      known="$(curl -s -f localhost:8500/v1/status/peers | jq -r .[] | awk -F ':' '{ print $1 }')"
      for peer in $(cat /var/run/weave/peers | grep 'master' | awk '{ print $1 }'); do
        if [ $(echo "$known" | grep "$peer" | wc -l) -eq "0" ]; then
          docker exec $consul_id consul join $peer
        fi
      done
      # kick out dead peers
      rm /tmp/consul_nodes /tmp/weave_peers || true
      echo "$(curl -s -f localhost:8500/v1/catalog/nodes | jq -r .[].Address | sort)" > /tmp/consul_nodes
      echo "$(cat /var/run/weave/peers | awk '{ print $1 }' | sort)" > /tmp/weave_peers
      dead=$(comm -23 /tmp/consul_nodes /tmp/weave_peers)
      for ip in $dead; do
        peer=$(curl -s -f localhost:8500/v1/catalog/nodes | jq -r ".[] | select(.Address==\"$ip\") | .Node")
        docker exec $consul_id consul force-leave $peer
      done
    fi

{% for key, value in services.iteritems() %}
    # ensure dns entry for {{ key }} exists
    if (status {{ key }} | grep "start/running" > /dev/null); then
      if (echo $WEAVE_DNS | grep -E "( |^|{{ hostname }}-){{ key }} [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ [a-f0-9]+ $NAME($| )" > /dev/null); then
        {{ key | regex_replace('[.-]', '_') | upper }}_NO_DNS=0
      else
        {{ key | regex_replace('[.-]', '_') | upper }}_NO_DNS=$(({{ key | regex_replace('[.-]', '_') | upper }}_NO_DNS + 1))
        echo "{{ key }} failed ${{ key | regex_replace('[.-]', '_') | upper }}_NO_DNS dns checks"
      fi
    else
      {{ key | regex_replace('[.-]', '_') | upper }}_NO_DNS=0
    fi

    if [ "${{ key | regex_replace('[.-]', '_') | upper }}_NO_DNS" -gt "2" ]; then
      echo "restarting {{ key }}"
      restart {{ key }}
      {{ key | regex_replace('[.-]', '_') | upper }}_NO_DNS=0
    fi

{% endfor %}
  done
end script

respawn
respawn limit unlimited
kill timeout 10