diff --git a/node_modules/kafka-node/README.md b/node_modules/kafka-node/README.md index 38041d0..a79e31d 100644 --- a/node_modules/kafka-node/README.md +++ b/node_modules/kafka-node/README.md @@ -51,6 +51,7 @@ Kafka-node is a Node.js client for Apache Kafka 0.9 and later. * Manage topic Offsets * SSL connections to brokers (Kafka 0.9+) * SASL/PLAIN Authentication (Kafka 0.10+) +* SASL/SCRAM-(256|512) AUthentication (Kafka .11+) * Consumer Groups managed by Kafka coordinator (Kafka 0.9+) * Connect directly to brokers (Kafka 0.9+) * Administrative APIs diff --git a/node_modules/kafka-node/docker/certs/ca-cert b/node_modules/kafka-node/docker/certs/ca-cert new file mode 100644 index 0000000..f8bef4d --- /dev/null +++ b/node_modules/kafka-node/docker/certs/ca-cert @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICsDCCAhmgAwIBAgIJAIyGnMEdl6tCMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMTYwNjIxMTQxMDA5WhcNNDMxMTA2MTQxMDA5WjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB +gQCebmAuz/u/vaGmm+rd+bSu4xDqWOD0nTlHgVcPps1mGNy8I+Je5ft4r8UXN6B4 +65rmJgXztg/HuBKU8mRO1VjwH7huA7NwsO+ve6i1eYQYBq3MtfxRj9gH4tE76147 +ET5FoFc8xn/2K09Lc/W5zhJHLeUDlYKP/SUiw5dFqKURMQIDAQABo4GnMIGkMB0G +A1UdDgQWBBTOi9ZSqUCykQrG0fwkA91MsC8HDTB1BgNVHSMEbjBsgBTOi9ZSqUCy +kQrG0fwkA91MsC8HDaFJpEcwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUt +U3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZIIJAIyGnMEd +l6tCMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEATpA/7+o740mEIVYY +xnZ3QR0TpW4SEudRBKoCH+obaSJH5LVxy59VohV8JeA0yQTRwn2SF1WQ5ZC5EuVO +daAfUwaYWwkGXrzS30KDuj5ospv6kUaIoSnHZvulFPQbqamO5mTsvoCZxKM79Ttz +EoM/vfy/Dehln4zt7ti1YdcU8ZM= +-----END CERTIFICATE----- diff --git a/node_modules/kafka-node/docker/certs/ca-cert.srl b/node_modules/kafka-node/docker/certs/ca-cert.srl new file mode 100644 index 0000000..8d4dc40 --- /dev/null +++ b/node_modules/kafka-node/docker/certs/ca-cert.srl @@ -0,0 +1 @@ +839412E1B55B2B15 diff --git a/node_modules/kafka-node/docker/certs/ca-key b/node_modules/kafka-node/docker/certs/ca-key new file mode 100644 index 0000000..0863842 --- /dev/null +++ b/node_modules/kafka-node/docker/certs/ca-key @@ -0,0 +1,18 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,6641B585EC61FD26 + +x6SWVdRDkrpsjLFwacn5vq1dOQwkwFvuGlJ9X1lf/lMYa//JUZtDRMi2Wea+WYhj +V31UEECgZHZxbzp74No5e4Bh05hN5id00Js0MRHMYDB3S0Hs7y75SQoAlqBqsGaC +VQX42xrNSzLW8jSvGxa+oX4YV54zdRdhmaNzEoWQ+qXMMdGezR1dRp6SGpwBnMv3 +UXlSYP+v6EFeGiUxhqatM9Sm6oXAcg71bJrc8VmHz6IPOeXm9+kRKZze/lQYmH0e +m1V4H46+kVBWGiStNiQa5IgJyDd2vAVO/p+/v3bbm9+U7I7LAW09NSpTVgTCb0Vv +RXEyBiBT1P48Aa9zNzk+DCOsSugdOjDj6HqJHs0mqC8D9wNuRmIEZm3D6SkZGf7C +XXJF0fDlt3pDXwOyJEC96SEFoYOef4h7Z0IEfAcRSOkhgrywZg9IzrAHbCbsX5Ij +/8NgGAaKDhef0p0niAXT+KmNV4AO8KHQKBVTga1jxcnCQEhBD4J8RqS7B6WQ+GqB +IE/fgV41JRyGOO6Hw/Y/0hjKbmYJTC9sjoUJ0a9QdVK7gZxGpd4Svo2XyVy2PfdR +DDVF4UJYKLaEWxXvUK1p7Dx+n2dqmkCoxJzomk0hXUfk/DtfeFD5++6STtAQJVgb +QUPWC3Enfmf79Fv8tbHPM7KClsd6Gc3pPLHC7UPIc3/Pgip5jrEFu8Jgm4kB2dJy +RQO/MB6LtU3nexdWaI6VDD3eTmrSNyRK/lCWLy8QHKK95+/xNmnZKfeXSm0D6ErR +Tm6nomuO0YleeWImoLzvA7WKXjwpqaUTzHnpZEsktYRAzUOIiG4ZlA== +-----END RSA PRIVATE KEY----- diff --git a/node_modules/kafka-node/docker/certs/cert-file b/node_modules/kafka-node/docker/certs/cert-file new file mode 100644 index 0000000..559d832 --- /dev/null +++ b/node_modules/kafka-node/docker/certs/cert-file @@ -0,0 +1,14 @@ +-----BEGIN NEW CERTIFICATE REQUEST----- +MIICoDCCAl4CAQAwbDEQMA4GA1UEBhMHVW5rbm93bjEQMA4GA1UECBMHVW5rbm93bjEQMA4GA1UE +BxMHVW5rbm93bjEQMA4GA1UEChMHVW5rbm93bjEQMA4GA1UECxMHVW5rbm93bjEQMA4GA1UEAxMH +VW5rbm93bjCCAbcwggEsBgcqhkjOOAQBMIIBHwKBgQD9f1OBHXUSKVLfSpwu7OTn9hG3UjzvRADD +Hj+AtlEmaUVdQCJR+1k9jVj6v8X1ujD2y5tVbNeBO4AdNG/yZmC3a5lQpaSfn+gEexAiwk+7qdf+ +t8Yb+DtX58aophUPBPuD9tPFHsMCNVQTWhaRMvZ1864rYdcq7/IiAxmd0UgBxwIVAJdgUI8VIwvM +spK5gqLrhAvwWBz1AoGBAPfhoIXWmz3ey7yrXDa4V7l5lK+7+jrqgvlXTAs9B4JnUVlXjrrUWU/m +cQcQgYC0SRZxI+hMKBYTt88JMozIpuE8FnqLVHyNKOCjrh4rs6Z1kW6jfwv6ITVi8ftiegEkO8yk +8b6oUZCJqIPf4VrlnwaSi2ZegHtVJWQBTDv+z0kqA4GEAAKBgDtgBe2nMFIxCeMGzzyiqyYxDWbp +BgQgsowJG3O1gq7grLDgokoo5nFd4YwoSGTFapnGKjU46y4jj7jkhMDn2oj9ufmxBoikUN0y39q0 +IdFiB/6y4eIXe1p61FVtKaFXn+RZWARRlFW3lTSOaLqnplRuqMGtw75M9y1c1k+I0oheoDAwLgYJ +KoZIhvcNAQkOMSEwHzAdBgNVHQ4EFgQUSH9fbKplQZ0gqwB0h3Kk0vC442wwCwYHKoZIzjgEAwUA +Ay8AMCwCFGHc9tgI+oUoFuuTnyQHp4EH14UFAhRhUfVRQdo4jBA/h9I5wVsxdfZSPw== +-----END NEW CERTIFICATE REQUEST----- diff --git a/node_modules/kafka-node/docker/certs/cert-signed b/node_modules/kafka-node/docker/certs/cert-signed new file mode 100644 index 0000000..d36ae2f --- /dev/null +++ b/node_modules/kafka-node/docker/certs/cert-signed @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDQTCCAqoCCQCDlBLhtVsrFTANBgkqhkiG9w0BAQUFADBFMQswCQYDVQQGEwJB +VTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0 +cyBQdHkgTHRkMB4XDTE2MDYyMTE0MTIyN1oXDTQzMTEwNjE0MTIyN1owbDEQMA4G +A1UEBhMHVW5rbm93bjEQMA4GA1UECBMHVW5rbm93bjEQMA4GA1UEBxMHVW5rbm93 +bjEQMA4GA1UEChMHVW5rbm93bjEQMA4GA1UECxMHVW5rbm93bjEQMA4GA1UEAxMH +VW5rbm93bjCCAbcwggEsBgcqhkjOOAQBMIIBHwKBgQD9f1OBHXUSKVLfSpwu7OTn +9hG3UjzvRADDHj+AtlEmaUVdQCJR+1k9jVj6v8X1ujD2y5tVbNeBO4AdNG/yZmC3 +a5lQpaSfn+gEexAiwk+7qdf+t8Yb+DtX58aophUPBPuD9tPFHsMCNVQTWhaRMvZ1 +864rYdcq7/IiAxmd0UgBxwIVAJdgUI8VIwvMspK5gqLrhAvwWBz1AoGBAPfhoIXW +mz3ey7yrXDa4V7l5lK+7+jrqgvlXTAs9B4JnUVlXjrrUWU/mcQcQgYC0SRZxI+hM +KBYTt88JMozIpuE8FnqLVHyNKOCjrh4rs6Z1kW6jfwv6ITVi8ftiegEkO8yk8b6o +UZCJqIPf4VrlnwaSi2ZegHtVJWQBTDv+z0kqA4GEAAKBgDtgBe2nMFIxCeMGzzyi +qyYxDWbpBgQgsowJG3O1gq7grLDgokoo5nFd4YwoSGTFapnGKjU46y4jj7jkhMDn +2oj9ufmxBoikUN0y39q0IdFiB/6y4eIXe1p61FVtKaFXn+RZWARRlFW3lTSOaLqn +plRuqMGtw75M9y1c1k+I0oheMA0GCSqGSIb3DQEBBQUAA4GBAGAK0H3OrcCe5K7c +yz9B/FBk40kjXhy0u2acVxaWh3+1avanzabo2NgbKv3q+Bkg6kNNzFAk5f2gbll2 +zT1lHZcaiQcpgHw7z1zuQ2OTrSUcpXYZXEqoJufgiBJkRZLOoRXQFodN7I8MLKHU +m9cl0wJGujKuydSiZJ20h/ecpwuS +-----END CERTIFICATE----- diff --git a/node_modules/kafka-node/docker/certs/client.truststore.jks b/node_modules/kafka-node/docker/certs/client.truststore.jks new file mode 100644 index 0000000..3535f7f Binary files /dev/null and b/node_modules/kafka-node/docker/certs/client.truststore.jks differ diff --git a/node_modules/kafka-node/docker/certs/server.keystore.jks b/node_modules/kafka-node/docker/certs/server.keystore.jks new file mode 100644 index 0000000..50ef683 Binary files /dev/null and b/node_modules/kafka-node/docker/certs/server.keystore.jks differ diff --git a/node_modules/kafka-node/docker/certs/server.truststore.jks b/node_modules/kafka-node/docker/certs/server.truststore.jks new file mode 100644 index 0000000..b3f2b22 Binary files /dev/null and b/node_modules/kafka-node/docker/certs/server.truststore.jks differ diff --git a/node_modules/kafka-node/docker/createTopic.js b/node_modules/kafka-node/docker/createTopic.js new file mode 100644 index 0000000..59f36a5 --- /dev/null +++ b/node_modules/kafka-node/docker/createTopic.js @@ -0,0 +1,30 @@ +'use strict'; + +const execa = require('execa'); +const assert = require('assert'); + +function createTopic (topicName, partitions, replicas, config = '') { + assert(topicName); + assert(partitions && partitions > 0); + assert(replicas && replicas > 0); + + const args = ['exec', '-T', 'kafka', 'bash', '-c']; + + if (process.env.KAFKA_VERSION === '0.9') { + const topic = `${topicName}:${partitions}:${replicas}`; + args.push(`KAFKA_CREATE_TOPICS=${topic} KAFKA_PORT=9092 /usr/bin/create-topics.sh`); + } else { + if (config) { + config = ` --config ${config}`; + } + args.push( + `/opt/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper:2181 --topic ${topicName} --partitions ${partitions} --replication-factor ${replicas} ${config}` + ); + } + + const createResult = execa('docker-compose', args); + // createResult.stdout.pipe(process.stdout); + return createResult; +} + +module.exports = createTopic; diff --git a/node_modules/kafka-node/docker/docker-compose.0.10.yml b/node_modules/kafka-node/docker/docker-compose.0.10.yml new file mode 100644 index 0000000..7ee338a --- /dev/null +++ b/node_modules/kafka-node/docker/docker-compose.0.10.yml @@ -0,0 +1,12 @@ +version: '2' +services: + kafka: + image: wurstmeister/kafka:0.10.2.1 + environment: + KAFKA_LISTENERS: "PLAINTEXT://:9092,SSL://:9093,SASL_PLAINTEXT://:9094" + KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://${KAFKA_ADVERTISED_HOST_NAME}:9092,SSL://${KAFKA_ADVERTISED_HOST_NAME}:9093,SASL_PLAINTEXT://${KAFKA_ADVERTISED_HOST_NAME}:9094" + KAFKA_SASL_ENABLED_MECHANISMS: "PLAIN" + KAFKA_SECURITY_INTER_BROKER_PROTOCOL: "PLAINTEXT" + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" + KAFKA_SUPER_USERS: "User:admin,User:broker" + KAFKA_OPTS: "-Djava.security.auth.login.config=/var/private/sasl/sasl.conf" diff --git a/node_modules/kafka-node/docker/docker-compose.0.11.yml b/node_modules/kafka-node/docker/docker-compose.0.11.yml new file mode 100644 index 0000000..676ba3d --- /dev/null +++ b/node_modules/kafka-node/docker/docker-compose.0.11.yml @@ -0,0 +1,16 @@ +version: '2' +services: + kafka: + image: wurstmeister/kafka:0.11.0.0 + environment: + KAFKA_LOG_MESSAGE_FORMAT_VERSION: "0.10.2" + KAFKA_LISTENERS: "PLAINTEXT://:9092,SSL://:9093,SASL_PLAINTEXT://:9094" + KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://${KAFKA_ADVERTISED_HOST_NAME}:9092,SSL://${KAFKA_ADVERTISED_HOST_NAME}:9093,SASL_PLAINTEXT://${KAFKA_ADVERTISED_HOST_NAME}:9094" + KAFKA_SASL_ENABLED_MECHANISMS: "PLAIN, SCRAM-SHA-512, SCRAM-SHA-256" + KAFKA_SECURITY_INTER_BROKER_PROTOCOL: "PLAINTEXT" + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" + KAFKA_SUPER_USERS: "User:admin,User:broker" + KAFKA_OPTS: "-Djava.security.auth.login.config=/var/private/sasl/sasl.conf" + volumes: + - ./docker/start-kafka.sh:/usr/bin/start-kafka.sh + diff --git a/node_modules/kafka-node/docker/docker-compose.0.9.yml b/node_modules/kafka-node/docker/docker-compose.0.9.yml new file mode 100644 index 0000000..ec28eed --- /dev/null +++ b/node_modules/kafka-node/docker/docker-compose.0.9.yml @@ -0,0 +1,4 @@ +version: '2' +services: + kafka: + image: wurstmeister/kafka:0.9.0.1 diff --git a/node_modules/kafka-node/docker/docker-compose.1.0.yml b/node_modules/kafka-node/docker/docker-compose.1.0.yml new file mode 100644 index 0000000..b6acf32 --- /dev/null +++ b/node_modules/kafka-node/docker/docker-compose.1.0.yml @@ -0,0 +1,16 @@ +version: '2' +services: + kafka: + image: wurstmeister/kafka:1.0.0 + environment: + KAFKA_LOG_MESSAGE_FORMAT_VERSION: "0.10.2" + KAFKA_LISTENERS: "PLAINTEXT://:9092,SSL://:9093,SASL_PLAINTEXT://:9094" + KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://${KAFKA_ADVERTISED_HOST_NAME}:9092,SSL://${KAFKA_ADVERTISED_HOST_NAME}:9093,SASL_PLAINTEXT://${KAFKA_ADVERTISED_HOST_NAME}:9094" + KAFKA_SASL_ENABLED_MECHANISMS: "PLAIN, SCRAM-SHA-512, SCRAM-SHA-256" + KAFKA_SECURITY_INTER_BROKER_PROTOCOL: "PLAINTEXT" + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" + KAFKA_SUPER_USERS: "User:admin,User:broker" + KAFKA_OPTS: "-Djava.security.auth.login.config=/var/private/sasl/sasl.conf" + volumes: + - ./docker/start-kafka.sh:/usr/bin/start-kafka.sh + diff --git a/node_modules/kafka-node/docker/docker-compose.1.1.yml b/node_modules/kafka-node/docker/docker-compose.1.1.yml new file mode 100644 index 0000000..8149f2f --- /dev/null +++ b/node_modules/kafka-node/docker/docker-compose.1.1.yml @@ -0,0 +1,16 @@ +version: '2' +services: + kafka: + image: wurstmeister/kafka:1.1.0 + environment: + KAFKA_LOG_MESSAGE_FORMAT_VERSION: "0.10.2" + KAFKA_LISTENERS: "PLAINTEXT://:9092,SSL://:9093,SASL_PLAINTEXT://:9094" + KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://${KAFKA_ADVERTISED_HOST_NAME}:9092,SSL://${KAFKA_ADVERTISED_HOST_NAME}:9093,SASL_PLAINTEXT://${KAFKA_ADVERTISED_HOST_NAME}:9094" + KAFKA_SASL_ENABLED_MECHANISMS: "PLAIN, SCRAM-SHA-512, SCRAM-SHA-256" + KAFKA_SECURITY_INTER_BROKER_PROTOCOL: "PLAINTEXT" + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" + KAFKA_SUPER_USERS: "User:admin,User:broker" + KAFKA_OPTS: "-Djava.security.auth.login.config=/var/private/sasl/sasl.conf" + volumes: + - ./docker/start-kafka.sh:/usr/bin/start-kafka.sh + diff --git a/node_modules/kafka-node/docker/docker-compose.2.0.yml b/node_modules/kafka-node/docker/docker-compose.2.0.yml new file mode 100644 index 0000000..a4db070 --- /dev/null +++ b/node_modules/kafka-node/docker/docker-compose.2.0.yml @@ -0,0 +1,16 @@ +version: '2' +services: + kafka: + image: wurstmeister/kafka:2.11-2.0.0 + environment: + KAFKA_LOG_MESSAGE_FORMAT_VERSION: "0.10.2" + KAFKA_LISTENERS: "PLAINTEXT://:9092,SSL://:9093,SASL_PLAINTEXT://:9094" + KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://${KAFKA_ADVERTISED_HOST_NAME}:9092,SSL://${KAFKA_ADVERTISED_HOST_NAME}:9093,SASL_PLAINTEXT://${KAFKA_ADVERTISED_HOST_NAME}:9094" + KAFKA_SASL_ENABLED_MECHANISMS: "PLAIN, SCRAM-SHA-512, SCRAM-SHA-256" + KAFKA_SECURITY_INTER_BROKER_PROTOCOL: "PLAINTEXT" + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" + KAFKA_SUPER_USERS: "User:admin,User:broker" + KAFKA_OPTS: "-Djava.security.auth.login.config=/var/private/sasl/sasl.conf" + volumes: + - ./docker/start-kafka.sh:/usr/bin/start-kafka.sh + diff --git a/node_modules/kafka-node/docker/sasl/sasl.conf b/node_modules/kafka-node/docker/sasl/sasl.conf new file mode 100644 index 0000000..62be2d2 --- /dev/null +++ b/node_modules/kafka-node/docker/sasl/sasl.conf @@ -0,0 +1,13 @@ +KafkaServer { + org.apache.kafka.common.security.plain.PlainLoginModule required + + username="broker" + password="broker" + user_kafkanode="kafkanode" + user_admin="admin" + user_broker="broker"; + + org.apache.kafka.common.security.scram.ScramLoginModule required + username="kafkanode" + password="kafkanode"; +}; diff --git a/node_modules/kafka-node/docker/start-kafka.sh b/node_modules/kafka-node/docker/start-kafka.sh new file mode 100755 index 0000000..fc829a4 --- /dev/null +++ b/node_modules/kafka-node/docker/start-kafka.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# +# FIXME KAFKA_OPTS with the SASL config causes a 'bad substitution' error in +# the sed commands below. Only seems to impact the 1.x images, not entirely +# sure why. +# +# Swapping KAFKA_OPTS out temporarily lets us work around the issue. +# +if [[ -n "$KAFKA_OPTS" ]]; then + TEMP_KAFKA_OPTS="$KAFKA_OPTS" + unset KAFKA_OPTS +fi + +if [[ -z "$KAFKA_PORT" ]]; then + export KAFKA_PORT=9092 +fi +if [[ -z "$KAFKA_ADVERTISED_PORT" && \ + -z "$KAFKA_LISTENERS" && \ + -z "$KAFKA_ADVERTISED_LISTENERS" && \ + -S /var/run/docker.sock ]]; then + export KAFKA_ADVERTISED_PORT=$(docker port `hostname` $KAFKA_PORT | sed -r "s/.*:(.*)/\1/g") +fi +if [[ -z "$KAFKA_BROKER_ID" ]]; then + if [[ -n "$BROKER_ID_COMMAND" ]]; then + export KAFKA_BROKER_ID=$(eval $BROKER_ID_COMMAND) + else + # By default auto allocate broker ID + export KAFKA_BROKER_ID=-1 + fi +fi +if [[ -z "$KAFKA_LOG_DIRS" ]]; then + export KAFKA_LOG_DIRS="/kafka/kafka-logs-$HOSTNAME" +fi +if [[ -z "$KAFKA_ZOOKEEPER_CONNECT" ]]; then + export KAFKA_ZOOKEEPER_CONNECT=$(env | grep ZK.*PORT_2181_TCP= | sed -e 's|.*tcp://||' | paste -sd ,) +fi + +if [[ -n "$KAFKA_HEAP_OPTS" ]]; then + sed -r -i "s/(export KAFKA_HEAP_OPTS)=\"(.*)\"/\1=\"$KAFKA_HEAP_OPTS\"/g" $KAFKA_HOME/bin/kafka-server-start.sh + unset KAFKA_HEAP_OPTS +fi + +if [[ -z "$KAFKA_ADVERTISED_HOST_NAME" && -n "$HOSTNAME_COMMAND" ]]; then + export KAFKA_ADVERTISED_HOST_NAME=$(eval $HOSTNAME_COMMAND) +fi + +if [[ -n "$RACK_COMMAND" && -z "$KAFKA_BROKER_RACK" ]]; then + export KAFKA_BROKER_RACK=$(eval $RACK_COMMAND) +fi + +#Issue newline to config file in case there is not one already +echo -e "\n" >> $KAFKA_HOME/config/server.properties + +for VAR in `env` +do + if [[ $VAR =~ ^KAFKA_ && ! $VAR =~ ^KAFKA_HOME && \ + ! $VAR =~ ^KAFKA_CREATE_TOPICS ]]; then + kafka_name=`echo "$VAR" | sed -r "s/KAFKA_(.*)=.*/\1/g" | tr '[:upper:]' '[:lower:]' | tr _ .` + env_var=`echo "$VAR" | sed -r "s/(.*)=.*/\1/g"` + if egrep -q "(^|^#)$kafka_name=" $KAFKA_HOME/config/server.properties; then + sed -r -i "s@(^|^#)($kafka_name)=(.*)@\2=${!env_var}@g" $KAFKA_HOME/config/server.properties #note that no config values may contain an '@' char + else + echo "$kafka_name=${!env_var}" >> $KAFKA_HOME/config/server.properties + fi + fi +done + +if [[ -n "$CUSTOM_INIT_SCRIPT" ]] ; then + eval $CUSTOM_INIT_SCRIPT +fi + +/opt/kafka/bin/kafka-configs.sh --zookeeper zookeeper:2181 --alter --add-config 'SCRAM-SHA-256=[password=kafkanode],SCRAM-SHA-512=[password=kafkanode]' --entity-type users --entity-name kafkanode & +create-topics.sh & +KAFKA_OPTS="$TEMP_KAFKA_OPTS" exec $KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/config/server.properties diff --git a/node_modules/kafka-node/lib/baseClient.js b/node_modules/kafka-node/lib/baseClient.js index 1c2ba24..22eff73 100644 --- a/node_modules/kafka-node/lib/baseClient.js +++ b/node_modules/kafka-node/lib/baseClient.js @@ -585,8 +585,8 @@ Client.prototype.invokeResponseCallback = function (socket, correlationId, resp) var handlers = this.unqueueCallback(socket, correlationId); if (handlers) { - var [decoder, cb] = handlers; - var result = decoder(resp); + var [decoder, cb, , decodeArgs] = handlers; + var result = decoder(resp, decodeArgs); if (result instanceof Error) { cb.call(this, result); } else { diff --git a/node_modules/kafka-node/lib/baseProducer.js b/node_modules/kafka-node/lib/baseProducer.js index 55854c7..12cb9f7 100644 --- a/node_modules/kafka-node/lib/baseProducer.js +++ b/node_modules/kafka-node/lib/baseProducer.js @@ -82,7 +82,13 @@ BaseProducer.prototype.connect = function () { // emiter... var self = this; this.ready = this.client.ready; - if (this.ready) self.emit('ready'); + if (this.ready) { + // Emit the ready event in next tick to give consumers a chance to set up + // a `ready` listener + setImmediate(function () { + self.emit('ready'); + }); + } this.client.on('ready', function () { if (!self.ready) { self.ready = true; diff --git a/node_modules/kafka-node/lib/kafkaClient.js b/node_modules/kafka-node/lib/kafkaClient.js index 1debd7b..55703a8 100644 --- a/node_modules/kafka-node/lib/kafkaClient.js +++ b/node_modules/kafka-node/lib/kafkaClient.js @@ -45,6 +45,7 @@ const DEFAULTS = { maxAsyncRequests: 10, noAckBatchOptions: null }; +const SUPPORTEDAUTH = ['PLAIN', 'SCRAM-SHA-256', 'SCRAM-SHA-512']; const KafkaClient = function (options) { EventEmitter.call(this); // Intentionally not calling Client to avoid constructor logic @@ -666,11 +667,18 @@ KafkaClient.prototype.initializeBroker = function (broker, callback) { KafkaClient.prototype.saslAuth = function (broker, callback) { const mechanism = this.options.sasl.mechanism.toUpperCase(); const apiVersion = broker.apiSupport ? broker.apiSupport.saslHandshake.usable : undefined; + const auth = this.options.sasl; + auth.mechanism = mechanism; + if (typeof apiVersion !== 'number') { callback(new errors.SaslAuthenticationError(null, 'Broker does not support SASL authentication')); return; } + if (!SUPPORTEDAUTH.includes(mechanism)) { + callback(new Error('unsupported SASL auth type: ' + mechanism.toUpperCase())); + return; + } async.waterfall( [ callback => { @@ -678,24 +686,50 @@ KafkaClient.prototype.saslAuth = function (broker, callback) { const correlationId = this.nextId(); const request = protocol.encodeSaslHandshakeRequest(this.clientId, correlationId, apiVersion, mechanism); - this.queueCallback(broker.socket, correlationId, [protocol.decodeSaslHandshakeResponse, callback]); broker.write(request); }, (enabledMechanisms, callback) => { logger.debug(`Sending SASL/${mechanism} authentication request to ${broker.socket.addr}`); - - const auth = this.options.sasl; + let request = null; + let decode = null; + let decodeArgs = { apiVersion }; const correlationId = this.nextId(); - const request = protocol.encodeSaslAuthenticateRequest(this.clientId, correlationId, apiVersion, auth); - let decode = protocol.decodeSaslAuthenticateResponse; + if (mechanism === 'PLAIN') { + request = protocol.encodeSaslPlainAuthRequest(this.clientId, correlationId, apiVersion, auth); + decode = protocol.decodeSaslPlainAuthRequest; + } + if (mechanism === 'SCRAM-SHA-256' || mechanism === 'SCRAM-SHA-512') { + let initialAuth = protocol.encodeScramInitialAuthRequest(this.clientId, correlationId, apiVersion, auth); + request = initialAuth.request; + auth.originalNonce = initialAuth.nonce; + decode = protocol.decodeScramInitialAuthRequest; + decodeArgs.nonce = initialAuth.nonce; + } + if (apiVersion === 0) { - decode = _.identity; broker.socket.saslAuthCorrelationId = correlationId; } - this.queueCallback(broker.socket, correlationId, [decode, callback]); + + this.queueCallback(broker.socket, correlationId, [decode, callback, null, decodeArgs]); broker.write(request); + }, + (serverReponse, callback) => { + if (mechanism === 'SCRAM-SHA-256' || mechanism === 'SCRAM-SHA-512') { + const correlationId = this.nextId(); + if (apiVersion === 0) { + broker.socket.saslAuthCorrelationId = correlationId; + } + + let { request, serverProof } = protocol.encodeScramFinalAuthRequest(this.clientId, correlationId, apiVersion, auth, serverReponse); + let decode = protocol.decodeScramFinalAuthRequest; + let decodeArgs = { apiVersion, serverProof }; + this.queueCallback(broker.socket, correlationId, [decode, callback, null, decodeArgs]); + broker.write(request); + } else { + callback(); + } } ], (error, authBytes) => { @@ -1371,8 +1405,11 @@ KafkaClient.prototype.handleReceivedData = function (socket) { } const resp = socket.buffer.slice(4, 4 + size); - this.invokeResponseCallback(socket, socket.saslAuthCorrelationId, resp); + + const socketId = socket.saslAuthCorrelationId; delete socket.saslAuthCorrelationId; + + this.invokeResponseCallback(socket, socketId, resp); socket.buffer.consume(size + 4); } else { return Client.prototype.handleReceivedData.call(this, socket); diff --git a/node_modules/kafka-node/lib/protocol/protocol.js b/node_modules/kafka-node/lib/protocol/protocol.js index c86cb7e..cd8610a 100644 --- a/node_modules/kafka-node/lib/protocol/protocol.js +++ b/node_modules/kafka-node/lib/protocol/protocol.js @@ -15,10 +15,22 @@ const MessageSizeTooLarge = require('../errors/MessageSizeTooLargeError'); const SaslAuthenticationError = require('../errors/SaslAuthenticationError'); const InvalidRequestError = require('../errors/InvalidRequestError'); const async = require('async'); +const scram = require('./scram'); var API_VERSION = 0; var REPLICA_ID = -1; var GROUPS_PROTOCOL_TYPE = 'consumer'; +var SCRAM_CONFIG = { + PLAIN: {}, + 'SCRAM-SHA-256': { + length: 32, + digest: 'sha256' + }, + 'SCRAM-SHA-512': { + length: 64, + digest: 'sha512' + } +}; function groupByTopic (payloads) { return payloads.reduce(function (out, p) { @@ -78,29 +90,7 @@ function decodeSaslHandshakeResponse (resp) { return new SaslAuthenticationError(errorCode, 'Handshake failed.'); } -function encodeSaslAuthenticateRequest (clientId, correlationId, apiVersion, saslOpts) { - // - // FIXME From the Kafka protocol docs: - // If SaslHandshakeRequest version is v0, a series of SASL client and server tokens - // corresponding to the mechanism are sent as opaque packets without wrapping the - // messages with Kafka protocol headers. If SaslHandshakeRequest version is v1, the - // SaslAuthenticate request/response are used, where the actual SASL tokens are - // wrapped in the Kafka protocol. - // - var username = saslOpts.username || ''; - var password = saslOpts.password || ''; - var authBytes = null; - if (saslOpts.mechanism.toUpperCase() === 'PLAIN') { - authBytes = - (new Buffermaker()) - .string(username).Int8(0) - .string(username).Int8(0) - .string(password) - .make(); - } else { - return new Error('unsupported SASL auth type: ' + saslOpts.mechanism.toUpperCase()); - } - +function encodeSaslAuthenticateRequest (clientId, correlationId, apiVersion, authBytes) { if (apiVersion === 0) { return encodeRequestWithLength(authBytes); } @@ -110,7 +100,10 @@ function encodeSaslAuthenticateRequest (clientId, correlationId, apiVersion, sas return encodeRequestWithLength(request.make()); } -function decodeSaslAuthenticateResponse (resp) { +function decodeSaslAuthenticateResponse (resp, apiVersion) { + if (apiVersion === 0) { + return resp.toString(); + } var errorCode = null; var errorMessage = null; var authBytes = null; @@ -121,8 +114,10 @@ function decodeSaslAuthenticateResponse (resp) { .word16bs('errorMessageLength') .tap(function (vars) { errorCode = vars.errorCode; - this.buffer('errorMessage', vars.errorMessageLength); - errorMessage = vars.errorMessage.toString(); + if (errorCode) { + this.buffer('errorMessage', vars.errorMessageLength); + errorMessage = vars.errorMessage.toString(); + } }) .word32bs('authBytesLength') .tap(function (vars) { @@ -132,9 +127,106 @@ function decodeSaslAuthenticateResponse (resp) { if (errorCode == null || errorCode === 0) { return authBytes; } + return new SaslAuthenticationError(errorCode, errorMessage); } +function encodeSaslPlainAuthRequest (clientId, correlationId, apiVersion, saslOpts) { + // + // FIXME From the Kafka protocol docs: + // If SaslHandshakeRequest version is v0, a series of SASL client and server tokens + // corresponding to the mechanism are sent as opaque packets without wrapping the + // messages with Kafka protocol headers. If SaslHandshakeRequest version is v1, the + // SaslAuthenticate request/response are used, where the actual SASL tokens are + // wrapped in the Kafka protocol. + // + var username = saslOpts.username || ''; + var password = saslOpts.password || ''; + var authBytes = (new Buffermaker()) + .string(username).Int8(0) + .string(username).Int8(0) + .string(password) + .make(); + + return encodeSaslAuthenticateRequest(clientId, correlationId, apiVersion, authBytes); +} + +function decodeSaslPlainAuthRequest (resp, encodeArgs) { + return decodeSaslAuthenticateResponse(resp, encodeArgs.apiVersion); +} + +function encodeScramInitialAuthRequest (clientId, correlationId, apiVersion, saslOpts) { + var username = saslOpts.username || ''; + var nonce = scram.nonce(); + var authBytes = (new Buffermaker()).string(`${scram.G2_HEADER}n=${username},r=${nonce}`).make(); + var request = encodeSaslAuthenticateRequest(clientId, correlationId, apiVersion, authBytes); + return { request, nonce }; +} + +function authBytesMap (authBytes) { + let responseObject = {}; + authBytes.split(',').map(str => { + const valuePair = str.split('='); + responseObject[valuePair[0]] = valuePair[1]; + }); + return responseObject; +} + +function decodeScramInitialAuthRequest (resp, encodeArgs) { + const authBytes = decodeSaslAuthenticateResponse(resp, encodeArgs.apiVersion); + if (authBytes instanceof Error) { + return authBytes; + } + + let responseObject = authBytesMap(authBytes); + responseObject.original = authBytes; + responseObject.i = Number.parseInt(responseObject.i); + // min of iteration is 4096 + if (responseObject.i < 4096) { + return new Error(`Server responded with invalid signature: ${responseObject.i}`); + } + + // server nonce should start with client's nonce + if (!responseObject.r.startsWith(encodeArgs.nonce)) { + return new Error('Server responded with invalid nonce'); + } + + return responseObject; +} + +function encodeScramFinalAuthRequest (clientId, correlationId, apiVersion, saslOpts, response) { + const { username, password, originalNonce } = saslOpts; + const withoutClientKey = `c=${Buffer.from(scram.G2_HEADER).toString('base64')},r=${response.r}`; + const normalizedPassword = scram.normalizePassword(password); + const authConfig = SCRAM_CONFIG[saslOpts.mechanism]; + // https://tools.ietf.org/html/rfc5802#section-3 + const spassword = scram.hi({ password: normalizedPassword, salt: Buffer.from(response.s, 'base64'), iterations: response.i }, authConfig); + const clientKey = scram.hmac(spassword, 'Client Key', authConfig); + const storedKey = scram.h(clientKey, authConfig); + const authMessage = `n=${username},r=${originalNonce},${response.original},${withoutClientKey}`; + const clientSignature = scram.hmac(storedKey, authMessage, authConfig); + const clientProof = (scram.xor(clientKey, clientSignature)).toString('base64'); + const serverkey = scram.hmac(spassword, 'Server Key', authConfig); + const serverProof = scram.hmac(serverkey, authMessage, authConfig).toString('base64'); + var authBytes = (new Buffermaker()).string(`${withoutClientKey},p=${clientProof}`).make(); + let request = encodeSaslAuthenticateRequest(clientId, correlationId, apiVersion, authBytes); + return { request, serverProof }; +} + +function decodeScramFinalAuthRequest (resp, encodeArgs) { + const authBytes = decodeSaslAuthenticateResponse(resp, encodeArgs.apiVersion); + if (authBytes instanceof Error) { + return authBytes; + } + + let responseObject = authBytesMap(authBytes); + let serverProof = encodeArgs.serverProof.replace(/=*$/, ''); + if (responseObject.v !== serverProof) { + return new Error('Server responded with invalid signature'); + } + return authBytes; +} + function encodeFetchRequest (maxWaitMs, minBytes) { return function encodeFetchRequest (clientId, correlationId, payloads) { return _encodeFetchRequest(clientId, correlationId, payloads, maxWaitMs, minBytes); @@ -1821,8 +1913,12 @@ function _decodeDescribeConfigsResponse (resp, apiVersion) { exports.encodeSaslHandshakeRequest = encodeSaslHandshakeRequest; exports.decodeSaslHandshakeResponse = decodeSaslHandshakeResponse; -exports.encodeSaslAuthenticateRequest = encodeSaslAuthenticateRequest; -exports.decodeSaslAuthenticateResponse = decodeSaslAuthenticateResponse; +exports.encodeSaslPlainAuthRequest = encodeSaslPlainAuthRequest; +exports.decodeSaslPlainAuthRequest = decodeSaslPlainAuthRequest; +exports.encodeScramInitialAuthRequest = encodeScramInitialAuthRequest; +exports.decodeScramInitialAuthRequest = decodeScramInitialAuthRequest; +exports.encodeScramFinalAuthRequest = encodeScramFinalAuthRequest; +exports.decodeScramFinalAuthRequest = decodeScramFinalAuthRequest; exports.encodeFetchRequest = encodeFetchRequest; exports.decodeFetchResponse = decodeFetchResponse; diff --git a/node_modules/kafka-node/lib/protocol/scram.js b/node_modules/kafka-node/lib/protocol/scram.js new file mode 100644 index 0000000..c99f28c --- /dev/null +++ b/node_modules/kafka-node/lib/protocol/scram.js @@ -0,0 +1,63 @@ +const crypto = require('crypto'); + +function hi (config, authConfig) { + const { password, salt, iterations } = config; + return crypto.pbkdf2Sync(password, salt, iterations, authConfig.length, authConfig.digest); +} + +function hmac (key, data, authConfig) { + return crypto.createHmac(authConfig.digest, key).update(data).digest(); +} + +function hash (data, authConfig) { + return crypto.createHash(authConfig.digest).update(data).digest(); +} + +function h (data, authConfig) { + return crypto + .createHash(authConfig.digest) + .update(data) + .digest(); +} + +function normalizePassword (password) { + return password.toString('utf-8'); +} + +function nonce () { + return crypto.randomBytes(16) + .toString('base64') + .replace(/\+/g, '-') // Convert '+' to '-' + .replace(/\//g, '_') // Convert '/' to '_' + .replace(/=+$/, '') // Remove ending '=' + .toString('ascii'); +} + +function xor (left, right) { + let leftBuffer = Buffer.from(left); + let rightBuffer = Buffer.from(right); + let leftLength = Buffer.byteLength(leftBuffer); + let rightLength = Buffer.byteLength(rightBuffer); + + if (leftLength !== rightLength) { + return new Error('Error while authentication (xor buffer length)'); + } + + let result = Buffer.alloc(leftLength); + for (let i = 0; i < leftLength; i++) { + result[i] = leftBuffer[i] ^ rightBuffer[i]; + } + + return result; +} + +const G2_HEADER = 'n,,'; + +exports.hi = hi; +exports.hmac = hmac; +exports.hash = hash; +exports.h = h; +exports.normalizePassword = normalizePassword; +exports.nonce = nonce; +exports.xor = xor; +exports.G2_HEADER = G2_HEADER; diff --git a/node_modules/kafka-node/test/.eslintrc.json b/node_modules/kafka-node/test/.eslintrc.json new file mode 100644 index 0000000..29c4553 --- /dev/null +++ b/node_modules/kafka-node/test/.eslintrc.json @@ -0,0 +1,8 @@ +{ + "env": { + "mocha": true + }, + "rules": { + "no-unused-expressions": "off" + } +} diff --git a/node_modules/kafka-node/test/assignment/test.range.js b/node_modules/kafka-node/test/assignment/test.range.js new file mode 100644 index 0000000..ff884e5 --- /dev/null +++ b/node_modules/kafka-node/test/assignment/test.range.js @@ -0,0 +1,118 @@ +'use strict'; + +const range = require('../../lib/assignment/range'); +const _ = require('lodash'); +const should = require('should'); + +describe('Range Assignment', function () { + const topicPartition = { + RebalanceTopic: ['0', '1', '2'], + RebalanceTest: ['0', '1', '2'] + }; + + const groupMembers = [ + { + subscription: ['RebalanceTopic', 'RebalanceTest'], + version: 0, + id: 'consumer1' + }, + { + subscription: ['RebalanceTopic', 'RebalanceTest'], + version: 0, + id: 'consumer2' + } + ]; + + it('should have required fields', function () { + range.should.have.property('assign').which.is.a.Function; + range.name.should.be.eql('range'); + range.version.should.be.eql(0); + }); + + it('should partition two topics of three partitions between two consumers', function (done) { + range.assign(topicPartition, groupMembers, function (error, result) { + should(error).be.empty; + const consumer1 = _.head(result); + consumer1.memberId.should.eql('consumer1'); + Object.keys(consumer1.topicPartitions).should.eql(['RebalanceTopic', 'RebalanceTest']); + consumer1.topicPartitions['RebalanceTest'].should.eql([0, 1]); + consumer1.topicPartitions['RebalanceTopic'].should.eql([0, 1]); + + const consumer2 = _.last(result); + consumer2.memberId.should.eql('consumer2'); + Object.keys(consumer2.topicPartitions).should.eql(['RebalanceTopic', 'RebalanceTest']); + consumer2.topicPartitions['RebalanceTest'].should.eql([2]); + consumer2.topicPartitions['RebalanceTopic'].should.eql([2]); + + done(); + }); + }); + + it('should partition two topics of three partitions between three consumers', function (done) { + const gm = groupMembers.slice(0); + gm.push({ + subscription: ['RebalanceTopic', 'RebalanceTest'], + version: 0, + id: 'consumer3' + }); + + range.assign(topicPartition, gm, function (error, result) { + should(error).be.empty; + const consumer1 = _.head(result); + consumer1.memberId.should.eql('consumer1'); + Object.keys(consumer1.topicPartitions).should.eql(['RebalanceTopic', 'RebalanceTest']); + consumer1.topicPartitions['RebalanceTest'].should.eql([0]); + consumer1.topicPartitions['RebalanceTopic'].should.eql([0]); + + const consumer2 = result[1]; + consumer2.memberId.should.eql('consumer2'); + Object.keys(consumer2.topicPartitions).should.eql(['RebalanceTopic', 'RebalanceTest']); + consumer2.topicPartitions['RebalanceTest'].should.eql([1]); + consumer2.topicPartitions['RebalanceTopic'].should.eql([1]); + + const consumer3 = _.last(result); + consumer3.memberId.should.eql('consumer3'); + Object.keys(consumer3.topicPartitions).should.eql(['RebalanceTopic', 'RebalanceTest']); + consumer3.topicPartitions['RebalanceTest'].should.eql([2]); + consumer3.topicPartitions['RebalanceTopic'].should.eql([2]); + + done(); + }); + }); + + it('should partition two topics of three partitions between four consumers', function (done) { + const gm = groupMembers.slice(0); + gm.push( + { + subscription: ['RebalanceTopic', 'RebalanceTest'], + version: 0, + id: 'consumer3' + }, + { + subscription: ['RebalanceTopic', 'RebalanceTest'], + version: 0, + id: 'consumer4' + } + ); + + range.assign(topicPartition, gm, function (error, result) { + should(error).be.empty; + const consumer1 = _.head(result); + consumer1.memberId.should.eql('consumer1'); + Object.keys(consumer1.topicPartitions).should.eql(['RebalanceTopic', 'RebalanceTest']); + consumer1.topicPartitions['RebalanceTest'].should.eql([0]); + consumer1.topicPartitions['RebalanceTopic'].should.eql([0]); + + const consumer2 = result[1]; + consumer2.memberId.should.eql('consumer2'); + Object.keys(consumer2.topicPartitions).should.eql(['RebalanceTopic', 'RebalanceTest']); + consumer2.topicPartitions['RebalanceTest'].should.eql([1]); + consumer2.topicPartitions['RebalanceTopic'].should.eql([1]); + + const consumer4 = _.last(result); + consumer4.memberId.should.eql('consumer4'); + Object.keys(consumer4.topicPartitions).should.eql([]); + done(); + }); + }); +}); diff --git a/node_modules/kafka-node/test/assignment/test.roundrobin.js b/node_modules/kafka-node/test/assignment/test.roundrobin.js new file mode 100644 index 0000000..17e9a66 --- /dev/null +++ b/node_modules/kafka-node/test/assignment/test.roundrobin.js @@ -0,0 +1,180 @@ +'use strict'; + +const roundRobin = require('../../lib/assignment/roundrobin'); +const _ = require('lodash'); +const should = require('should'); + +describe('Round Robin Assignment', function () { + const topicPartition = { + 'RebalanceTopic': [ + '0', + '1', + '2' + ], + 'RebalanceTest': [ + '0', + '1', + '2' + ] + }; + + it('should have required fields', function () { + roundRobin.should.have.property('assign').which.is.a.Function; + roundRobin.name.should.be.eql('roundrobin'); + roundRobin.version.should.be.eql(0); + }); + + it('should distribute two topics three partitions to two consumers ', function (done) { + const groupMembers = [ + { + 'subscription': [ + 'RebalanceTopic', + 'RebalanceTest' + ], + 'version': 0, + 'id': 'consumer1' + }, + { + 'subscription': [ + 'RebalanceTopic', + 'RebalanceTest' + ], + 'version': 0, + 'id': 'consumer2' + } + ]; + + roundRobin.assign(topicPartition, groupMembers, function (error, assignment) { + should(error).be.empty; + const consumer1 = _.head(assignment); + consumer1.memberId.should.eql('consumer1'); + Object.keys(consumer1.topicPartitions).should.eql(['RebalanceTopic', 'RebalanceTest']); + consumer1.topicPartitions['RebalanceTest'].should.eql(['1']); + consumer1.topicPartitions['RebalanceTopic'].should.eql(['0', '2']); + + const consumer2 = _.last(assignment); + consumer2.memberId.should.eql('consumer2'); + Object.keys(consumer2.topicPartitions).should.eql(['RebalanceTopic', 'RebalanceTest']); + consumer2.topicPartitions['RebalanceTest'].should.eql(['0', '2']); + consumer2.topicPartitions['RebalanceTopic'].should.eql(['1']); + done(); + }); + }); + + it('should distribute two topics three partitions to three consumers', function (done) { + const groupMembers = [ + { + 'subscription': [ + 'RebalanceTopic', + 'RebalanceTest' + ], + 'version': 0, + 'id': 'consumer1' + }, + { + 'subscription': [ + 'RebalanceTopic', + 'RebalanceTest' + ], + 'version': 0, + 'id': 'consumer3' + }, + { + 'subscription': [ + 'RebalanceTopic', + 'RebalanceTest' + ], + 'version': 0, + 'id': 'consumer2' + } + ]; + + roundRobin.assign(topicPartition, groupMembers, function (error, assignment) { + should(error).be.empty; + assignment = _.sortBy(assignment, 'memberId'); + + const consumer1 = _.head(assignment); + consumer1.memberId.should.eql('consumer1'); + Object.keys(consumer1.topicPartitions).should.eql(['RebalanceTopic', 'RebalanceTest']); + consumer1.topicPartitions['RebalanceTest'].should.eql(['0']); + consumer1.topicPartitions['RebalanceTopic'].should.eql(['0']); + + const consumer2 = assignment[1]; + consumer2.memberId.should.eql('consumer2'); + Object.keys(consumer2.topicPartitions).should.eql(['RebalanceTopic', 'RebalanceTest']); + consumer2.topicPartitions['RebalanceTest'].should.eql(['1']); + consumer2.topicPartitions['RebalanceTopic'].should.eql(['1']); + + const consumer3 = _.last(assignment); + consumer3.memberId.should.eql('consumer3'); + Object.keys(consumer3.topicPartitions).should.eql(['RebalanceTopic', 'RebalanceTest']); + consumer3.topicPartitions['RebalanceTest'].should.eql(['2']); + consumer3.topicPartitions['RebalanceTopic'].should.eql(['2']); + done(); + }); + }); + + it('should distribute two topics three partitions to four consumers', function (done) { + const groupMembers = [ + { + 'subscription': [ + 'RebalanceTopic', + 'RebalanceTest' + ], + 'version': 0, + 'id': 'consumer1' + }, + { + 'subscription': [ + 'RebalanceTopic', + 'RebalanceTest' + ], + 'version': 0, + 'id': 'consumer3' + }, + { + 'subscription': [ + 'RebalanceTopic', + 'RebalanceTest' + ], + 'version': 0, + 'id': 'consumer2' + }, + { + 'subscription': [ + 'RebalanceTopic', + 'RebalanceTest' + ], + 'version': 0, + 'id': 'consumer4' + } + ]; + + roundRobin.assign(topicPartition, groupMembers, function (error, assignment) { + should(error).be.empty; + assignment = _.sortBy(assignment, 'memberId'); + + const consumer1 = _.head(assignment); + consumer1.memberId.should.eql('consumer1'); + Object.keys(consumer1.topicPartitions).should.eql(['RebalanceTopic', 'RebalanceTest']); + consumer1.topicPartitions['RebalanceTest'].should.eql(['1']); + consumer1.topicPartitions['RebalanceTopic'].should.eql(['0']); + + const consumer2 = assignment[1]; + consumer2.memberId.should.eql('consumer2'); + Object.keys(consumer2.topicPartitions).should.eql(['RebalanceTopic', 'RebalanceTest']); + consumer2.topicPartitions['RebalanceTest'].should.eql(['2']); + consumer2.topicPartitions['RebalanceTopic'].should.eql(['1']); + + const consumer3 = assignment[2]; + consumer3.memberId.should.eql('consumer3'); + Object.keys(consumer3.topicPartitions).should.eql(['RebalanceTopic']); + consumer3.topicPartitions['RebalanceTopic'].should.eql(['2']); + + const consumer4 = _.last(assignment); + consumer4.memberId.should.eql('consumer4'); + Object.keys(consumer4.topicPartitions).should.eql(['RebalanceTest']); + done(); + }); + }); +}); diff --git a/node_modules/kafka-node/test/helpers/Childrearer.js b/node_modules/kafka-node/test/helpers/Childrearer.js new file mode 100644 index 0000000..58ba9f4 --- /dev/null +++ b/node_modules/kafka-node/test/helpers/Childrearer.js @@ -0,0 +1,107 @@ +var EventEmitter = require('events').EventEmitter; +var util = require('util'); +var debug = require('debug')('kafka-node:Test-Childrearer'); +var fork = require('child_process').fork; +var async = require('async'); +var _ = require('lodash'); + +function Childrearer (forkPath) { + EventEmitter.call(this); + this.children = []; + this.id = 0; + this.forkPath = forkPath || 'test/helpers/child-hlc'; +} + +util.inherits(Childrearer, EventEmitter); + +Childrearer.prototype.setVerifier = function (topic, groupId, verify) { + this.topic = topic; + this.groupId = groupId; + this.verify = verify; +}; + +Childrearer.prototype.nextId = function () { + return ++this.id; +}; + +Childrearer.prototype.closeAll = function (callback) { + async.each(this.children, function (child, callback) { + child.once('exit', function () { + callback(null); + }); + child.kill(); + }, callback); +}; + +Childrearer.prototype.kill = function (numberOfChildren, callback) { + var children = _.sampleSize(this.children, numberOfChildren); + this._killEachChild(children, callback); +}; + +Childrearer.prototype.killLast = function (callback) { + var child = _.last(this.children); + this._killEachChild([child], callback); +}; + +Childrearer.prototype.killFirst = function (callback) { + var child = _.head(this.children); + this._killEachChild([child], callback); +}; + +Childrearer.prototype._killEachChild = function (children, callback) { + var self = this; + async.each(children, function (child, callback) { + child.once('exit', function (code, signal) { + debug('child %s killed %d %s', this._childNum, code, signal); + _.pull(self.children, this); + callback(); + }); + child.kill(); + }, callback); +}; + +Childrearer.prototype.raise = function (children, callback, waitTime) { + var newChildren = _.times(children, _.bind(this._raiseChild, this)); + + this.children = this.children.concat(newChildren); + + if (callback) { + async.series([ + function (callback) { + async.each(newChildren, function (child, callback) { + child.once('message', function (data) { + if (data.event === 'registered') { + callback(null); + } else { + callback(new Error('unregistered event: ' + data.event)); + } + }); + }, callback); + }, + + function (callback) { + if (waitTime) { + setTimeout(callback, waitTime); + } else { + callback(); + } + }], callback + ); + } +}; + +Childrearer.prototype._raiseChild = function () { + var self = this; + var childNumber = this.nextId(); + debug('forking child %d', childNumber); + var child = fork(this.forkPath, ['--groupId=' + this.groupId, '--topic=' + this.topic, '--consumerId=' + 'child_' + childNumber]); + child._childNum = childNumber; + child.on('message', function (data) { + if (data.message) { + self.verify.call(this, data); + } + }); + return child; +}; + +module.exports = Childrearer; diff --git a/node_modules/kafka-node/test/helpers/EventCounter.js b/node_modules/kafka-node/test/helpers/EventCounter.js new file mode 100644 index 0000000..004833c --- /dev/null +++ b/node_modules/kafka-node/test/helpers/EventCounter.js @@ -0,0 +1,39 @@ +'use strict'; + +class EventCounter { + constructor () { + this.events = {}; + } + + /** + * Creates an event counter with optional expected number and callback. + * + * @param {String} eventName - The name of the event counter. + * @param {Number} limit - The number of events after which the callback + * should be called. + * @param {Function} callback - The callback to envoke when the expected + * number is reached. + * @returns {Function} - A function that can be called to increment the + * coutner and collect the result. + */ + createEventCounter (eventName, limit, callback) { + if (!limit) { + limit = Number.POSITIVE_INFINITY; + } + this.events[eventName] = { + count: 0, + events: [] + }; + return function () { + this.events[eventName].count++; + this.events[eventName].events.push(arguments); + if (this.events[eventName].count === limit) { + if (callback) { + callback(null, this.events[eventName]); + } + } + }.bind(this); + } +} + +module.exports = EventCounter; diff --git a/node_modules/kafka-node/test/helpers/child-cg-kafka-client.js b/node_modules/kafka-node/test/helpers/child-cg-kafka-client.js new file mode 100644 index 0000000..90ade3e --- /dev/null +++ b/node_modules/kafka-node/test/helpers/child-cg-kafka-client.js @@ -0,0 +1,71 @@ +'use strict'; + +var kafka = require('../../'); +var ConsumerGroup = kafka.ConsumerGroup; +var argv = require('optimist').argv; +var topic = argv.topic || 'topic1'; +var options = { + kafkaHost: '127.0.0.1:9092', + autoCommit: true, + fetchMaxWaitMs: 1000, + fetchMaxBytes: 1024 * 1024, + sessionTimeout: 8000, + heartbeatInterval: 250, + retryMinTimeout: 250, + versions: { + requestTimeout: 150 + } +}; +var debug = require('debug')('kafka-node:Child-ConsumerGroup'); + +if (argv.groupId) { + options.groupId = argv.groupId; +} + +if (argv.consumerId) { + options.id = argv.consumerId; +} + +var consumer = new ConsumerGroup(options, [topic]); + +consumer.on('message', function (message) { + var out = { + id: consumer.client.clientId, + message: message + }; + process.send(out); +}); + +consumer.on('error', function (err) { + debug('error', err); +}); + +consumer.on('rebalanced', function () { + debug('%s rebalanced!', consumer.client.clientId); + sendEvent('rebalanced'); +}); + +consumer.on('rebalancing', function () { + debug('%s is rebalancing', consumer.client.clientId); +}); + +function sendEvent (event) { + process.send({ + id: consumer.client.clientId, + event: event + }); +} + +function close (signal) { + return function () { + debug('closing the consumer (%s) [%s].', signal, consumer.client.clientId); + consumer.close(true, function () { + process.exit(); + }); + }; +} + +process.once('SIGINT', close('SIGINT')); +process.once('SIGTERM', close('SIGTERM')); +process.once('SIGABRT', close('SIGABRT')); +process.once('disconnect', close('disconnect')); diff --git a/node_modules/kafka-node/test/helpers/sendMessage.js b/node_modules/kafka-node/test/helpers/sendMessage.js new file mode 100644 index 0000000..2e5db6f --- /dev/null +++ b/node_modules/kafka-node/test/helpers/sendMessage.js @@ -0,0 +1,22 @@ +'use strict'; + +const KafkaClient = require('../../lib/kafkaClient'); +const HighLevelProducer = require('../../lib/highLevelProducer'); + +function sendMessage (message, topic, done) { + var client = new KafkaClient({ kafkaHost: '127.0.0.1:9092' }); + var producer = new HighLevelProducer(client, { requireAcks: 1 }); + + client.on('connect', function () { + producer.send([{ topic: topic, messages: message, attributes: 0 }], function (error) { + if (error) { + done(error); + } else { + done(null); + } + producer.close(function () {}); + }); + }); +} + +module.exports = sendMessage; diff --git a/node_modules/kafka-node/test/helpers/sendMessageEach.js b/node_modules/kafka-node/test/helpers/sendMessageEach.js new file mode 100644 index 0000000..bfd0c05 --- /dev/null +++ b/node_modules/kafka-node/test/helpers/sendMessageEach.js @@ -0,0 +1,31 @@ +'use strict'; + +const KafkaClient = require('../../lib/kafkaClient'); +const HighLevelProducer = require('../../lib/highLevelProducer'); +const async = require('async'); +const uuid = require('uuid'); + +function sendMessage (message, topic, done, attributes = 0) { + var client = new KafkaClient({ kafkaHost: '127.0.0.1:9092' }); + var producer = new HighLevelProducer(client, { requireAcks: 1 }); + + client.on('connect', function () { + async.each( + message, + function (message, callback) { + producer.send([{ topic: topic, messages: message, key: uuid.v4(), attributes, timestamp: Date.now() }], callback); + }, + function (error) { + if (error) { + done(error); + } else { + producer.close(function () { + done(null); + }); + } + } + ); + }); +} + +module.exports = sendMessage; diff --git a/node_modules/kafka-node/test/manual.gracefulexit.js b/node_modules/kafka-node/test/manual.gracefulexit.js new file mode 100644 index 0000000..ddb152c --- /dev/null +++ b/node_modules/kafka-node/test/manual.gracefulexit.js @@ -0,0 +1,15 @@ +// Run this test with: +// mocha test/manual.gracefulexit.js --no-exit + +var Client = require('../lib/client'); + +describe('Client', function () { + describe('#close', function () { + it('should close gracefully', function (done) { + var client = new Client(); + client.on('ready', function () { + client.close(done); + }); + }); + }); +}); diff --git a/node_modules/kafka-node/test/mocha.opts b/node_modules/kafka-node/test/mocha.opts new file mode 100644 index 0000000..58c077b --- /dev/null +++ b/node_modules/kafka-node/test/mocha.opts @@ -0,0 +1,6 @@ +--require should +--reporter spec +--ui bdd +--slow 2s +--timeout 4s +--recursive diff --git a/node_modules/kafka-node/test/mocks/mockClient.js b/node_modules/kafka-node/test/mocks/mockClient.js new file mode 100644 index 0000000..87c91c7 --- /dev/null +++ b/node_modules/kafka-node/test/mocks/mockClient.js @@ -0,0 +1,37 @@ +'use strict'; + +var util = require('util'); +var EventEmitter = require('events').EventEmitter; + +function FakeClient () { + EventEmitter.call(this); + + this.topicExists = function (topics, cb) { + setImmediate(cb); + }; + this.refreshMetadata = function (topicNames, cb) { + setImmediate(cb); + }; + this.sendOffsetCommitRequest = function (groupId, commits, cb) { + setImmediate(cb); + }; + this.sendFetchRequest = function (consumer, payloads, fetchMaxWaitMs, fetchMinBytes, maxTickMessages) {}; + this.sendOffsetFetchRequest = function (groupId, payloads, cb) { + setImmediate(cb); + }; + this.sendOffsetRequest = function (payloads, cb) { + setImmediate(cb); + }; + this.addTopics = function (topics, cb) { + setImmediate(cb); + }; + this.removeTopicMetadata = function (topics, cb) { + setImmediate(cb); + }; + this.close = function (cb) { + setImmediate(cb); + }; +} +util.inherits(FakeClient, EventEmitter); + +module.exports = FakeClient; diff --git a/node_modules/kafka-node/test/mocks/mockSocket.js b/node_modules/kafka-node/test/mocks/mockSocket.js new file mode 100644 index 0000000..a6bd72d --- /dev/null +++ b/node_modules/kafka-node/test/mocks/mockSocket.js @@ -0,0 +1,23 @@ +var util = require('util'); +var EventEmitter = require('events').EventEmitter; + +function FakeSocket () { + EventEmitter.call(this); + + this.unref = function () {}; + + this.end = function () { + var self = this; + setImmediate(function () { + self.emit('end'); + }); + }; + this.close = function () {}; + this.setKeepAlive = function () {}; + this.destroy = function () {}; + this.write = function () {}; +} + +util.inherits(FakeSocket, EventEmitter); + +module.exports = FakeSocket; diff --git a/node_modules/kafka-node/test/test.admin.js b/node_modules/kafka-node/test/test.admin.js new file mode 100644 index 0000000..f22b4a9 --- /dev/null +++ b/node_modules/kafka-node/test/test.admin.js @@ -0,0 +1,213 @@ +'use strict'; + +const Admin = require('../lib/admin'); +const ConsumerGroup = require('../lib/consumerGroup'); +const KafkaClient = require('../lib/kafkaClient'); +const uuid = require('uuid'); +const should = require('should'); + +describe('Admin', function () { + describe('#listGroups', function () { + const createTopic = require('../docker/createTopic'); + let admin, consumer; + const topic = uuid.v4(); + const groupId = 'test-group-id'; + + before(function (done) { + createTopic(topic, 1, 1).then(function () { + consumer = new ConsumerGroup( + { + kafkaHost: 'localhost:9092', + groupId: groupId + }, + topic + ); + consumer.once('connect', function () { + done(); + }); + }); + }); + + after(function (done) { + consumer.close(done); + }); + + it('should return a list of consumer groups', function (done) { + admin = new Admin(consumer.client); + admin.listGroups(function (error, res) { + res.should.have.keys(groupId); + res[groupId].should.eql('consumer'); + done(error); + }); + }); + }); + + describe('#describeGroups', function () { + const createTopic = require('../docker/createTopic'); + let admin, consumer; + const topic = uuid.v4(); + const groupId = 'test-group-id'; + + before(function (done) { + createTopic(topic, 1, 1).then(function () { + consumer = new ConsumerGroup( + { + kafkaHost: 'localhost:9092', + groupId: groupId + }, + topic + ); + consumer.once('connect', function () { + done(); + }); + }); + }); + + after(function (done) { + consumer.close(done); + }); + + it('should describe a list of consumer groups', function (done) { + admin = new Admin(consumer.client); + admin.describeGroups([groupId], function (error, res) { + res.should.have.keys(groupId); + res[groupId].should.have.property('members').with.lengthOf(1); + res[groupId].should.have.property('state', 'Stable'); + done(error); + }); + }); + + it('should return empty members if consumer group doesnt exist', function (done) { + admin = new Admin(consumer.client); + const nonExistentGroup = 'non-existent-group'; + admin.describeGroups([nonExistentGroup], function (error, res) { + res.should.have.keys(nonExistentGroup); + res[nonExistentGroup].should.have.property('members').with.lengthOf(0); + res[nonExistentGroup].should.have.property('state', 'Dead'); + done(error); + }); + }); + }); + + describe('#describeConfigs', function () { + const createTopic = require('../docker/createTopic'); + let admin, client; + const topicName = uuid.v4(); + + before(function (done) { + if (['0.9', '0.10'].includes(process.env.KAFKA_VERSION)) { + this.skip(); + } + + createTopic(topicName, 1, 1).then(function () { + client = new KafkaClient({ kafkaHost: 'localhost:9092' }); + admin = new Admin(client); + admin.once('ready', done); + }); + }); + + after(function (done) { + if (client) { + client.close(done); + } else { + done(); + } + }); + + it('should describe a list of topic configs', function (done) { + const request = { + resourceType: admin.RESOURCE_TYPES.topic, + resourceName: topicName, + configNames: [] + }; + const payload = { + includeSynonyms: false, + resources: [request] + }; + admin.describeConfigs(payload, function (error, res) { + res.should.be.instanceof(Array); + res.length.should.be.exactly(1); + const entries = res[0]; + entries.should.have.property('resourceType').and.exactly('2'); + entries.should.have.property('resourceName').and.exactly(topicName); + entries.should.have.property('configEntries'); + entries.configEntries.length.should.be.greaterThan(1); + done(error); + }); + }); + + it('should describe a list of broker configs for a specific broker id', function (done) { + const brokerName = '1001'; + const request = { + resourceType: admin.RESOURCE_TYPES.broker, + resourceName: brokerName, + configNames: [] + }; + const payload = { + includeSynonyms: false, + resources: [request] + }; + admin.describeConfigs(payload, function (error, res) { + res.should.be.instanceof(Array); + res.length.should.be.exactly(1); + const entries = res[0]; + entries.should.have.property('resourceType').and.exactly('4'); + entries.should.have.property('resourceName').and.exactly(brokerName); + entries.should.have.property('configEntries'); + entries.configEntries.length.should.be.greaterThan(1); + done(error); + }); + }); + + it('should return an error if the resource (topic) doesnt exist', function (done) { + const request = { + resourceType: admin.RESOURCE_TYPES.topic, + resourceName: '', + configNames: [] + }; + const payload = { + includeSynonyms: false, + resources: [request] + }; + admin.describeConfigs(payload, function (error, res) { + error.should.have.property('message').and.containEql('InvalidTopic'); + done(); + }); + }); + + it('should return an error if the resource (broker) doesnt exist', function (done) { + const brokerId = '9999'; + const request = { + resourceType: admin.RESOURCE_TYPES.broker, + resourceName: brokerId, + configNames: [] + }; + const payload = { + includeSynonyms: false, + resources: [request] + }; + admin.describeConfigs(payload, function (error, res) { + should.not.exist(res); + error.should.have.property('message').and.containEql('No broker with id ' + brokerId); + done(); + }); + }); + + it('should return error for invalid resource type', function (done) { + const request = { + resourceType: 25, + resourceName: topicName, + configNames: [] + }; + const payload = { + includeSynonyms: false, + resources: [request] + }; + admin.describeConfigs(payload, function (error, res) { + should.not.exist(res); + error.should.have.property('message').and.equal(`Unexpected resource type 25 for resource ${topicName}`); + done(); + }); + }); + }); +}); diff --git a/node_modules/kafka-node/test/test.baseProducer.js b/node_modules/kafka-node/test/test.baseProducer.js new file mode 100644 index 0000000..7b8e7d9 --- /dev/null +++ b/node_modules/kafka-node/test/test.baseProducer.js @@ -0,0 +1,304 @@ +'use strict'; + +const BaseProducer = require('../lib/baseProducer'); +const ConsumerGroup = require('../lib/consumerGroup'); +const KafkaClient = require('../lib/kafkaClient'); +const Client = require('./mocks/mockClient'); +const uuid = require('uuid'); +const sinon = require('sinon'); +const async = require('async'); +const should = require('should'); + +describe('BaseProducer', function () { + describe('ready event', function () { + const KAFKA_HOST = 'localhost:9092'; + let client; + before(function () { + client = new KafkaClient({ + kafkaHost: KAFKA_HOST + }); + }); + + it('can listen on the ready event before the client is connected', function (done) { + const producer = new BaseProducer(client, {}, BaseProducer.PARTITIONER_TYPES.default); + producer.once('ready', function () { + should(producer.ready).be.true; + done(); + }); + }); + + it('can listen on the ready event after the client is connected', function (done) { + should(client.ready).be.true; + const producer = new BaseProducer(client, {}, BaseProducer.PARTITIONER_TYPES.default); + producer.once('ready', function () { + should(producer.ready).be.true; + done(); + }); + }); + }); + + describe('encoding and decoding key attribute', function () { + const KAFKA_HOST = 'localhost:9092'; + let consumerGroup, topic, producer; + beforeEach(function (done) { + topic = uuid.v4(); + + const createTopic = require('../docker/createTopic'); + + async.series( + [ + function (callback) { + createTopic(topic, 1, 1) + .then(function () { + callback(null); + }) + .catch(error => callback(error)); + }, + function (callback) { + const client = new KafkaClient({ + kafkaHost: KAFKA_HOST + }); + + producer = new BaseProducer(client, {}, BaseProducer.PARTITIONER_TYPES.default); + producer.once('ready', function () { + callback(null); + }); + }, + function (callback) { + consumerGroup = new ConsumerGroup( + { + kafkaHost: KAFKA_HOST, + groupId: uuid.v4() + }, + topic + ); + consumerGroup.once('connect', function () { + callback(null); + }); + } + ], + done + ); + }); + + describe('gzip compression', function () { + afterEach(function () { + consumerGroup.on('error', function () {}); + }); + + it('kafkaClient', function (done) { + const messageValue = uuid.v4(); + const time = Date.now(); + producer.send( + [ + { + topic: topic, + messages: messageValue, + key: 'myKeyIsHere', + attributes: 1, + timestamp: time + } + ], + function (error) { + if (error) { + done(error); + } + } + ); + + consumerGroup.on('message', function (message) { + console.log(message); + message.key.should.be.exactly('myKeyIsHere'); + message.value.should.be.exactly(messageValue); + should(message.timestamp.getTime()).be.exactly(time); + done(); + }); + }); + }); + + afterEach(function (done) { + producer.close(); + consumerGroup.close(done); + }); + + it('verify key string value makes it into the message', function (done) { + producer.send( + [ + { + topic: topic, + messages: 'this is my message', + key: 'myKeyIsHere' + } + ], + function (error) { + if (error) { + done(error); + } + } + ); + + consumerGroup.on('message', function (message) { + message.key.should.be.exactly('myKeyIsHere'); + done(); + }); + }); + + it('verify empty key string value makes it into the message', function (done) { + producer.send( + [ + { + topic: topic, + messages: 'this is my message', + key: '' + } + ], + function (error) { + if (error) { + done(error); + } + } + ); + + consumerGroup.on('message', function (message) { + message.key.should.be.exactly(''); + done(); + }); + }); + + it('verify key value of 0 makes it into the message', function (done) { + producer.send( + [ + { + topic: topic, + messages: 'this is my message', + key: 0 + } + ], + function (error) { + if (error) { + done(error); + } + } + ); + + consumerGroup.on('message', function (message) { + message.key.should.be.exactly('0'); + done(); + }); + }); + + it('verify key value of null makes it into the message as null', function (done) { + producer.send( + [ + { + topic: topic, + messages: 'this is my message', + key: null + } + ], + function (error) { + if (error) { + done(error); + } + } + ); + + consumerGroup.on('message', function (message) { + should(message.key).be.null; + done(); + }); + }); + + it('verify key value of undefined makes it into the message as null', function (done) { + producer.send( + [ + { + topic: topic, + messages: 'this is my message', + key: undefined + } + ], + function (error) { + if (error) { + done(error); + } + } + ); + + consumerGroup.on('message', function (message) { + should(message.key).be.null; + done(); + }); + }); + + it('verify key value of buffer makes it into the message as untouched buffer', function (done) { + const keyBuffer = Buffer.from('testing123'); + producer.send( + [ + { + topic: topic, + messages: 'this is my message', + key: keyBuffer + } + ], + function (error) { + if (error) { + done(error); + } + } + ); + + consumerGroup.options.keyEncoding = 'buffer'; + + consumerGroup.on('message', function (message) { + should(message.key).not.be.empty; + keyBuffer.equals(message.key).should.be.true; + done(); + }); + }); + }); + + describe('On Brokers Changed', function () { + it('should emit error when refreshMetadata fails', function (done) { + const fakeClient = new Client(); + fakeClient.topicMetadata = {}; + + const producer = new BaseProducer(fakeClient, {}, BaseProducer.PARTITIONER_TYPES.default); + + producer.once('error', function (error) { + error.should.be.an.instanceOf(Error); + error.message.should.be.exactly('boo'); + done(); + }); + + const myError = new Error('boo'); + const refreshMetadataStub = sinon.stub(fakeClient, 'refreshMetadata').yields(myError); + + fakeClient.emit('brokersChanged'); + + sinon.assert.calledWith(refreshMetadataStub, []); + }); + + it('should call client.refreshMetadata when brokerChanges', function (done) { + const fakeClient = new Client(); + + fakeClient.topicMetadata = { + MyTopic: [0], + YourTopic: [0, 1, 2] + }; + + const producer = new BaseProducer(fakeClient, {}, BaseProducer.PARTITIONER_TYPES.default); + + producer.once('error', done); + + const refreshMetadataStub = sinon.stub(fakeClient, 'refreshMetadata').yields(null); + + fakeClient.emit('brokersChanged'); + + fakeClient.topicMetadata.should.have.property('MyTopic'); + fakeClient.topicMetadata.should.have.property('YourTopic'); + sinon.assert.calledWith(refreshMetadataStub, ['MyTopic', 'YourTopic']); + done(); + }); + }); +}); diff --git a/node_modules/kafka-node/test/test.consumer.js b/node_modules/kafka-node/test/test.consumer.js new file mode 100644 index 0000000..c951f7e --- /dev/null +++ b/node_modules/kafka-node/test/test.consumer.js @@ -0,0 +1,692 @@ +'use strict'; + +var libPath = process.env['KAFKA_COV'] ? '../lib-cov/' : '../lib/'; +var Consumer = require(libPath + 'consumer'); +var Producer = require(libPath + 'producer'); +var Offset = require(libPath + 'offset'); +var Client = require(libPath + 'kafkaClient'); +var should = require('should'); +var uuid = require('uuid'); +var TopicsNotExistError = require(libPath + 'errors').TopicsNotExistError; +var FakeClient = require('./mocks/mockClient'); +var InvalidConfigError = require('../lib/errors/InvalidConfigError'); + +const createTopic = require('../docker/createTopic'); +const sendMessage = require('./helpers/sendMessage'); +const _ = require('lodash'); + +var client, producer, offset; + +var TOPIC_POSTFIX = '_test_' + Date.now(); +var EXISTS_TOPIC_1 = '_exists_1' + TOPIC_POSTFIX; +var EXISTS_TOPIC_2 = '_exists_2' + TOPIC_POSTFIX; +var EXISTS_GZIP = '_exists_gzip'; // + TOPIC_POSTFIX +var EXISTS_SNAPPY = '_exists_snappy'; // + TOPIC_POSTFIX + +// Compression-friendly to be interesting +var SNAPPY_MESSAGE = new Array(20).join('snappy'); + +var host = process.env['KAFKA_TEST_HOST'] || ''; +function noop () { + console.log(arguments); +} + +describe('Consumer', function () { + describe('#setOffset', function () { + var client, consumer; + + beforeEach(function () { + client = new FakeClient(); + + consumer = new Consumer(client, [{ topic: 'fake-topic' }]); + + clearTimeout(consumer.checkPartitionOwnershipInterval); + consumer.payloads = [ + { topic: 'fake-topic', partition: '0', offset: 0, maxBytes: 1048576, metadata: 'm' }, + { topic: 'fake-topic', partition: '1', offset: 0, maxBytes: 1048576, metadata: 'm' } + ]; + }); + + it('should setOffset correctly given partition is a string', function () { + consumer.setOffset('fake-topic', '0', 86); + consumer.payloads[0].offset.should.be.eql(86); + + consumer.setOffset('fake-topic', '1', 23); + consumer.payloads[1].offset.should.be.eql(23); + }); + + it('should setOffset correctly if given partition is a number', function () { + consumer.setOffset('fake-topic', 0, 67); + consumer.payloads[0].offset.should.be.eql(67); + + consumer.setOffset('fake-topic', 1, 98); + consumer.payloads[1].offset.should.be.eql(98); + }); + }); + + describe('validate topics', function () { + function validateThrowsInvalidConfigError (topics) { + should.throws(function () { + var client = new FakeClient(host); + // eslint-disable-next-line no-new + new Consumer(client, topics); + }, InvalidConfigError); + } + + function validateDoesNotThrowInvalidConfigError (topics) { + should.doesNotThrow(function () { + var client = new FakeClient(host); + // eslint-disable-next-line no-new + new Consumer(client, topics); + }, InvalidConfigError); + } + + it('should throw an error on invalid topic partitions', function () { + validateThrowsInvalidConfigError([ + { + topic: 'another-topic' + }, + { + topic: 'my-fake-topic', + partition: '7' + } + ]); + + validateThrowsInvalidConfigError([ + { + topic: 'another-topic', + partition: 3 + }, + + { + topic: 'my-fake-topic', + partition: {} + } + ]); + + validateThrowsInvalidConfigError([ + { + topic: 'my-fake-topic', + partition: false + } + ]); + }); + + it('should not throw an error for valid topic partition', function () { + validateDoesNotThrowInvalidConfigError([ + { + topic: 'fake-topic', + partition: 7 + }, + { + topic: 'another-topic' + } + ]); + validateDoesNotThrowInvalidConfigError([ + { + topic: 'another-topic' + }, + { + topic: 'fake-topic', + partition: 0 + } + ]); + }); + }); + + describe('Compression', function () { + let topic, messages; + + before(function () { + if (process.env.KAFKA_VERSION === '0.9') { + this.skip(); + } + + topic = uuid.v4(); + messages = _.times(10, uuid.v4); + return createTopic(topic, 1, 1, 'compression.type=gzip').then(function () { + return new Promise(function (resolve, reject) { + sendMessage(messages, topic, function (error) { + if (error) { + return reject(error); + } + resolve(); + }); + }); + }); + }); + + it('should not throw offsetOutOfRange error', function (done) { + const client = new Client({ kafkaHost: '127.0.0.1:9092' }); + const consumer = new Consumer(client, [ + { + topic, + partition: 0 + } + ]); + consumer.on('offsetOutOfRange', done); + consumer.on('message', function (message) { + if (_.pull(messages, message.value).length === 0) { + setTimeout(function () { + consumer.close(done); + }, 50); + } + }); + }); + }); + + describe('validate groupId', function () { + function validateThrowsInvalidConfigError (groupId) { + should.throws(function () { + var client = new FakeClient(host); + // eslint-disable-next-line no-new + new Consumer(client, [{ topic: EXISTS_TOPIC_2 }], { groupId: groupId }); + }, InvalidConfigError); + } + + function validateDoesNotThrowInvalidConfigError (groupId) { + should.doesNotThrow(function () { + var client = new FakeClient(host); + // eslint-disable-next-line no-new + new Consumer(client, [{ topic: EXISTS_TOPIC_2 }], { groupId: groupId }); + }); + } + + it('should throws an error on invalid group IDs', function () { + validateThrowsInvalidConfigError('myGroupId:12345'); + validateThrowsInvalidConfigError('myGroupId,12345'); + validateThrowsInvalidConfigError('myGroupId"12345"'); + validateThrowsInvalidConfigError('myGroupId?12345'); + }); + + it('should not throw on valid group IDs', function () { + validateDoesNotThrowInvalidConfigError('myGroupId.12345'); + validateDoesNotThrowInvalidConfigError('something_12345'); + validateDoesNotThrowInvalidConfigError('myGroupId-12345'); + }); + }); + + function offsetOutOfRange (topic, consumer) { + topic.maxNum = 2; + offset.fetch([topic], function (err, offsets) { + if (err) consumer.emit('error', err); + var min = Math.min.apply(null, offsets[topic.topic][topic.partition]); + consumer.setOffset(topic.topic, topic.partition, min); + }); + } + + [ + { + name: 'PLAINTEXT Consumer' + }, + { + name: 'SSL Consumer', + sslOptions: { + rejectUnauthorized: false + }, + suiteTimeout: 30000 + } + ].forEach(function (testParameters) { + var sslOptions = testParameters.sslOptions || undefined; + var suiteTimeout = testParameters.suiteTimeout || null; + var suiteName = testParameters.name; + const kafkaHost = '127.0.0.1:' + (sslOptions == null ? '9092' : '9093'); + + function createClient () { + return new Client({ kafkaHost, sslOptions }); + } + + describe(suiteName, function () { + before(function (done) { + if (suiteTimeout) { + this.timeout(suiteTimeout); + } + client = createClient(); + producer = new Producer(client); + offset = new Offset(client); + producer.on('ready', function () { + producer.createTopics([EXISTS_TOPIC_1, EXISTS_TOPIC_2, EXISTS_GZIP, EXISTS_SNAPPY], true, function (err) { + if (err) return done(err); + producer.send( + [ + { topic: EXISTS_TOPIC_2, messages: 'hello kafka' }, + { topic: EXISTS_GZIP, messages: 'hello gzip', attributes: 1 }, + { topic: EXISTS_SNAPPY, messages: SNAPPY_MESSAGE, attributes: 2 } + ], + done + ); + }); + }); + }); + + after(function (done) { + client.close(done); + }); + + afterEach(function () { + client.removeAllListeners(); + }); + + describe('events', function () { + it('should emit message when get new message', function (done) { + if (suiteTimeout) { + this.timeout(suiteTimeout); + } + var topics = [{ topic: EXISTS_TOPIC_2 }]; + var options = { autoCommit: false, groupId: '_groupId_1_test' }; + var consumer = new Consumer(client, topics, options); + var count = 0; + consumer.on('error', noop); + consumer.on('offsetOutOfRange', function (topic) { + offsetOutOfRange(topic, this); + }); + consumer.on('message', function (message) { + message.topic.should.equal(EXISTS_TOPIC_2); + message.value.should.equal('hello kafka'); + message.highWaterOffset.should.be.above(0); + message.partition.should.equal(0); + offset.commit('_groupId_1_test', [message], function (err) { + if (count++ === 0) done(err); + }); + }); + }); + + it('should decode gzip messages', function (done) { + if (suiteTimeout) { + this.timeout(suiteTimeout); + } + var topics = [{ topic: EXISTS_GZIP }]; + var options = { autoCommit: false, groupId: '_groupId_gzip_test' }; + var client = createClient(); + var consumer = new Consumer(client, topics, options); + var count = 0; + consumer.on('error', noop); + consumer.on('offsetOutOfRange', function (topic) { + offsetOutOfRange(topic, this); + }); + consumer.on('message', function (message) { + message.topic.should.equal(EXISTS_GZIP); + message.value.should.equal('hello gzip'); + message.highWaterOffset.should.be.above(0); + offset.commit('_groupId_gzip_test', [message], function (err) { + if (count++ === 0) { + consumer.close(function () { + done(err); + }); + } + }); + }); + }); + + it('should decode snappy messages', function (done) { + if (suiteTimeout) { + this.timeout(suiteTimeout); + } + var topics = [{ topic: EXISTS_SNAPPY }]; + var options = { autoCommit: false, groupId: '_groupId_snappy_test' }; + var client = createClient(); + var consumer = new Consumer(client, topics, options); + var count = 0; + consumer.on('error', noop); + consumer.on('offsetOutOfRange', function (topic) { + offsetOutOfRange(topic, this); + }); + consumer.once('message', function (message) { + message.topic.should.equal(EXISTS_SNAPPY); + message.value.should.equal(SNAPPY_MESSAGE); + message.highWaterOffset.should.be.above(0); + offset.commit('_groupId_snappy_test', [message], function (err) { + if (count++ === 0) { + consumer.close(function () { + done(err); + }); + } + }); + }); + }); + + it('should emit error when topic not exists', function (done) { + var topics = [{ topic: '_not_exist_topic_1_test' }]; + var consumer = new Consumer(client, topics); + consumer.on('error', function (error) { + error.should.be.an + .instanceOf(TopicsNotExistError) + .and.have.property('message', 'The topic(s) _not_exist_topic_1_test do not exist'); + done(); + }); + }); + + it('should emit offsetOutOfRange when offset out of range', function (done) { + var topics = [{ topic: EXISTS_TOPIC_1, offset: 100 }]; + var options = { fromOffset: true, autoCommit: false }; + var count = 0; + var client = createClient(); + var consumer = new Consumer(client, topics, options); + consumer.on('offsetOutOfRange', function (topic) { + topic.topic.should.equal(EXISTS_TOPIC_1); + topic.partition.should.equal(0); + topic.message.should.equal('OffsetOutOfRange'); + if (count++ === 0) { + consumer.close(function () { + done(); + }); + } + }); + consumer.on('error', noop); + }); + }); + + describe('#addTopics', function () { + describe('when topic need to added not exist', function () { + it('should return error', function (done) { + var options = { autoCommit: false, groupId: '_groupId_1_test' }; + var topics = ['_not_exist_topic_1_test']; + var consumer = new Consumer(client, [], options); + consumer.on('error', noop); + consumer.addTopics(topics, function (err) { + err.topics.length.should.equal(1); + err.topics.should.eql(topics); + done(); + }); + }); + + it('should return error when using payload as well', function (done) { + var options = { autoCommit: false, groupId: '_groupId_1_test' }; + var topics = [{ topic: '_not_exist_topic_1_test', offset: 42 }]; + var consumer = new Consumer(client, [], options); + consumer.on('error', noop); + consumer.addTopics( + topics, + function (err) { + err.topics.length.should.equal(1); + err.topics.should.eql( + topics.map(function (p) { + return p.topic; + }) + ); + done(); + }, + true + ); + }); + }); + + describe('when topic need to added exist', function () { + it('should added successfully', function (done) { + var options = { autoCommit: false, groupId: '_groupId_addTopics_test' }; + var topics = [EXISTS_TOPIC_2]; + var consumer = new Consumer(client, [], options); + consumer.on('error', noop); + consumer.addTopics(topics, function (err, data) { + data.should.eql(topics); + consumer.payloads.some(function (p) { + return p.topic === topics[0]; + }).should.be.ok; + done(err); + }); + }); + + it('should add with given offset', function (done) { + var options = { autoCommit: false, groupId: '_groupId_addTopics_test' }; + var topics = [{ topic: EXISTS_TOPIC_2, offset: 42 }]; + var consumer = new Consumer(client, [], options); + consumer.on('error', noop); + consumer.addTopics( + topics, + function (err, data) { + data.should.eql( + topics.map(function (p) { + return p.topic; + }) + ); + consumer.payloads.some(function (p) { + return p.topic === topics[0].topic && p.offset === topics[0].offset; + }).should.be.ok; + done(err); + }, + true + ); + }); + }); + + describe('#removeTopics', function () { + it('should remove topics successfully', function (done) { + var options = { autoCommit: false, groupId: '_groupId_addTopics_test' }; + var topics = [{ topic: EXISTS_TOPIC_2 }, { topic: EXISTS_TOPIC_1 }]; + var consumer = new Consumer(client, topics, options); + consumer.on('error', noop); + + consumer.payloads.length.should.equal(2); + consumer.removeTopics(EXISTS_TOPIC_2, function (err, removed) { + removed.should.equal(1); + consumer.payloads.length.should.equal(1); + done(err); + }); + }); + }); + + describe('#setOffset', function () { + it('should update the offset in consumer', function () { + var options = { autoCommit: false, groupId: '_groupId_addTopics_test' }; + var topics = [{ topic: EXISTS_TOPIC_2 }]; + var consumer = new Consumer(client, topics, options); + consumer.on('error', noop); + + consumer.setOffset(EXISTS_TOPIC_2, 0, 100); + consumer.payloads + .filter(function (p) { + return p.topic === EXISTS_TOPIC_2; + })[0] + .offset.should.equal(100); + }); + }); + + describe('#commit', function () { + it('should commit offset of current topics', function (done) { + var topics = [{ topic: EXISTS_TOPIC_2 }]; + var options = { autoCommit: false, groupId: '_groupId_commit_test' }; + + var client = new Client(); + var consumer = new Consumer(client, topics, options); + var count = 0; + consumer.on('error', noop); + consumer.on('offsetOutOfRange', function (topic) { + offsetOutOfRange(topic, this); + }); + consumer.on('message', function (message) { + consumer.commit(true, function (err) { + if (!err && count++ === 0) done(err); + }); + }); + }); + }); + + describe('#buildPayloads', function () { + it('should set default config on the topic', function () { + var defaults = { + groupId: 'kafka-node-group', + autoCommit: true, + autoCommitIntervalMs: 5000, + encoding: 'utf8', + fetchMaxWaitMs: 100, + fetchMinBytes: 1, + fetchMaxBytes: 1024 * 1024, + fromOffset: false + }; + + var consumer = new Consumer(client, []); + consumer.options.should.eql(defaults); + + var topic = consumer.buildPayloads([{ topic: 'topic' }])[0]; + topic.partition.should.equal(0); + topic.offset.should.equal(0); + topic.metadata.should.equal('m'); + topic.maxBytes.should.equal(1024 * 1024); + }); + + it('should support custom options', function () { + var options = { + groupId: 'custom-group', + autoCommit: false, + autoCommitIntervalMs: 1000, + fetchMaxWaitMs: 200, + fetchMinBytes: 1, + fetchMaxBytes: 1024, + fromOffset: false + }; + var consumer = new Consumer(client, [], options); + consumer.options.should.equal(options); + var topic = consumer.buildPayloads([{ topic: 'topic' }])[0]; + topic.partition.should.equal(0); + topic.offset.should.equal(0); + topic.metadata.should.equal('m'); + topic.maxBytes.should.equal(1024); + }); + + it('should return right payloads when arguments is string', function () { + var consumer = new Consumer(client, []); + var topics = ['topic']; + var topic = consumer.buildPayloads(topics)[0]; + topic.should.be.an.instanceof(Object); + topic.partition.should.equal(0); + topic.offset.should.equal(0); + topic.metadata.should.equal('m'); + topic.maxBytes.should.equal(1024 * 1024); + }); + }); + + describe('#close', function () { + it('should close the consumer', function (done) { + var client = new Client(); + var topics = [{ topic: EXISTS_TOPIC_2 }]; + var options = { autoCommit: false, groupId: '_groupId_close_test' }; + + var consumer = new Consumer(client, topics, options); + consumer.once('message', function (message) { + consumer.close(function (err) { + done(err); + }); + }); + }); + + it('should commit the offset if force', function (done) { + var client = new Client(); + var topics = [{ topic: EXISTS_TOPIC_2 }]; + var force = true; + var options = { autoCommit: false, groupId: '_groupId_close_test' }; + + var consumer = new Consumer(client, topics, options); + consumer.once('message', function (message) { + consumer.close(force, function (err) { + done(err); + }); + }); + }); + }); + + describe('#pauseTopics|resumeTopics', function () { + const suffix = uuid.v4(); + const topic1 = `_test_topic_1_${suffix}`; + const topic2 = `_test_topic_2_${suffix}`; + + const createTopic = require('../docker/createTopic'); + + before(function () { + return Promise.all([createTopic(topic1, 2, 1), createTopic(topic2, 2, 1)]).then(function () { + return new Promise(function (resolve, reject) { + producer.send([{ topic: topic2, messages: 'hello kafka' }], function (error) { + if (error) { + return reject(error); + } + resolve(); + }); + }); + }); + }); + + it('should pause or resume the topics', function (done) { + var client = createClient(); + var topics = [ + { topic: topic1, partition: 0 }, + { topic: topic1, partition: 1 }, + { topic: topic2, partition: 0 }, + { topic: topic2, partition: 1 } + ]; + var consumer = new Consumer(client, topics, {}); + consumer.on('error', function () {}); + + function normalize (p) { + return { topic: p.topic, partition: p.partition }; + } + function compare (p1, p2) { + return p1.topic === p2.topic ? p1.partition - p2.partition : p1.topic > p2.topic; + } + + consumer.payloads.should.eql(topics); + consumer.pauseTopics([topic1, { topic: topic2, partition: 0 }]); + consumer.payloads.map(normalize).should.eql([{ topic: topic2, partition: 1 }]); + + consumer.resumeTopics([{ topic: topic1, partition: 0 }]); + + const actual = consumer.payloads.map(normalize).sort(compare); + const expected = [{ topic: topic2, partition: 1 }, { topic: topic1, partition: 0 }].sort(compare); + + actual.should.eql(expected); + + consumer.pausedPayloads + .map(normalize) + .sort(compare) + .should.eql([{ topic: topic1, partition: 1 }, { topic: topic2, partition: 0 }]); + + consumer.resumeTopics([topic1, topic2]); + + for (const tp of topics) { + consumer.payloads.should.containEql(tp); + } + + consumer.pausedPayloads.should.eql([]); + consumer.once('message', function () { + consumer.close(true, done); + }); + }); + }); + + describe('Consumer', function () { + before(function (done) { + if (suiteTimeout) { + this.timeout(suiteTimeout); + } + client = createClient(); + producer = new Producer(client); + offset = new Offset(client); + producer.on('ready', function () { + producer.createTopics([EXISTS_TOPIC_1, EXISTS_TOPIC_2, EXISTS_GZIP, EXISTS_SNAPPY], false, function ( + err, + created + ) { + if (err) return done(err); + + function useNewTopics () { + producer.send( + [ + { topic: EXISTS_TOPIC_2, messages: 'hello kafka' }, + { topic: EXISTS_GZIP, messages: 'hello gzip', attributes: 1 }, + { topic: EXISTS_SNAPPY, messages: SNAPPY_MESSAGE, attributes: 2 } + ], + done + ); + } + // Ensure leader selection happened + setTimeout(useNewTopics, 1000); + }); + }); + }); + }); + + after(function (done) { + client.close(done); + }); + }); + }); + }); +}); diff --git a/node_modules/kafka-node/test/test.consumerGroup.js b/node_modules/kafka-node/test/test.consumerGroup.js new file mode 100644 index 0000000..325465e --- /dev/null +++ b/node_modules/kafka-node/test/test.consumerGroup.js @@ -0,0 +1,1349 @@ +'use strict'; + +const sinon = require('sinon'); +const should = require('should'); +const ConsumerGroup = require('../lib/consumerGroup'); +const sendMessage = require('./helpers/sendMessage'); +const sendMessageEach = require('./helpers/sendMessageEach'); +const _ = require('lodash'); +const proxyquire = require('proxyquire').noCallThru(); +const EventEmitter = require('events').EventEmitter; +const createTopic = require('../docker/createTopic'); +const uuid = require('uuid'); +const async = require('async'); +const BrokerWrapper = require('../lib/wrapper/BrokerWrapper'); +const FakeSocket = require('./mocks/mockSocket'); +const { BrokerNotAvailableError } = require('../lib/errors'); + +describe('ConsumerGroup', function () { + describe('#constructor', function () { + var ConsumerGroup; + var fakeClient = sinon.stub().returns(new EventEmitter()); + + beforeEach(function () { + ConsumerGroup = proxyquire('../lib/consumerGroup', { + './kafkaClient': fakeClient + }); + }); + + it('should throw an error if using an invalid outOfRangeOffset', function () { + [true, false, '', 0, 1, 'blah'].forEach(offset => { + should.throws(() => { + // eslint-disable-next-line no-new + new ConsumerGroup({ + outOfRangeOffset: offset + }); + }); + }); + }); + + it('should not throw an error if using an valid outOfRangeOffset', function () { + ['earliest', 'latest', 'none'].forEach(offset => { + should.doesNotThrow(() => { + // eslint-disable-next-line no-new + new ConsumerGroup( + { + outOfRangeOffset: offset, + connectOnReady: false + }, + 'TestTopic' + ); + }); + }); + }); + + it('should throw an error if using an invalid fromOffset', function () { + [true, false, '', 0, 1, 'blah'].forEach(offset => { + should.throws(() => { + // eslint-disable-next-line no-new + new ConsumerGroup({ + fromOffset: offset + }); + }); + }); + }); + + it('should not throw an error if using an valid fromOffset', function () { + ['earliest', 'latest', 'none'].forEach(offset => { + should.doesNotThrow(() => { + // eslint-disable-next-line no-new + new ConsumerGroup( + { + fromOffset: offset, + connectOnReady: false + }, + 'TestTopic' + ); + }); + }); + }); + + it('should throw assertion error if using empty topic array', function () { + should.throws(() => { + // eslint-disable-next-line no-new + new ConsumerGroup({}, []); + }); + }); + + it('should not throw assertion error if using valid topic array', function () { + should.doesNotThrow(() => { + // eslint-disable-next-line no-new + new ConsumerGroup({}, ['test']); + }); + }); + }); + + describe('Compression', function () { + function verifyMessagesAndOffsets (topic, messages, done) { + const consumer = new ConsumerGroup( + { kafkaHost: '127.0.0.1:9092', groupId: uuid.v4(), fromOffset: 'earliest' }, + topic + ); + let verifyOffset = 0; + const allMessages = messages.slice(0); + consumer.on('offsetOutOfRange', done); + consumer.on('message', function (message) { + message.offset.should.be.equal(verifyOffset++); + message.partition.should.be.equal(0); + message.value.should.be.equal(allMessages[message.offset]); + if (_.pull(messages, message.value).length === 0) { + setTimeout(function () { + consumer.close(done); + }, 50); + } + }); + } + + describe('with topic config enabled send batch', function () { + let topic, messages; + before(function () { + if (process.env.KAFKA_VERSION === '0.9') { + this.skip(); + } + + topic = uuid.v4(); + messages = _.times(25, function () { + return new Array(100).join(Math.random().toString(36)); + }); + return createTopic(topic, 1, 1, 'compression.type=gzip').then(function () { + return new Promise(function (resolve, reject) { + sendMessage(messages, topic, function (error) { + if (error) { + return reject(error); + } + resolve(); + }); + }); + }); + }); + + it('should not throw offsetOutOfRange error', function (done) { + verifyMessagesAndOffsets(topic, messages, done); + }); + }); + + describe('with topic config enabled send each', function () { + let topic, messages; + before(function () { + if (process.env.KAFKA_VERSION === '0.9') { + this.skip(); + } + + topic = uuid.v4(); + messages = _.times(25, function () { + return new Array(100).join(Math.random().toString(36)); + }); + return createTopic(topic, 1, 1, 'compression.type=gzip').then(function () { + return new Promise(function (resolve, reject) { + sendMessageEach(messages, topic, function (error) { + if (error) { + return reject(error); + } + resolve(); + }); + }); + }); + }); + + it('should not throw offsetOutOfRange error', function (done) { + verifyMessagesAndOffsets(topic, messages, done); + }); + }); + }); + + describe('Topic partition change detection', function () { + let ConsumerGroup = null; + let consumerGroup = null; + let sandbox = null; + + const fakeClient = new EventEmitter(); + fakeClient.loadMetadataForTopics = function () {}; + + const FakeClient = function () { + return fakeClient; + }; + + beforeEach(function () { + sandbox = sinon.sandbox.create(); + ConsumerGroup = proxyquire('../lib/consumerGroup', { + './kafkaClient': FakeClient + }); + + consumerGroup = new ConsumerGroup( + { + host: 'gibberish', + connectOnReady: false + }, + 'TestTopic' + ); + }); + + afterEach(function () { + sandbox.restore(); + }); + + describe('#scheduleTopicPartitionCheck', function () { + let clock; + beforeEach(function () { + clock = sandbox.useFakeTimers(); + }); + + it('should not schedule check if consumer is closed', function () { + const cgMock = sandbox.mock(consumerGroup); + + consumerGroup.closed = true; + consumerGroup.isLeader = true; + + cgMock.expects('_checkTopicPartitionChange').never(); + cgMock.expects('commit').never(); + cgMock.expects('leaveGroup').never(); + cgMock.expects('connect').never(); + consumerGroup.scheduleTopicPartitionCheck(); + clock.tick(30000); + + cgMock.verify(); + }); + + it('should not run check if consumer is closed and scheduled', function () { + const cgMock = sandbox.mock(consumerGroup); + + consumerGroup.closed = false; + consumerGroup.isLeader = true; + + cgMock.expects('_checkTopicPartitionChange').never(); + cgMock.expects('commit').never(); + cgMock.expects('leaveGroup').never(); + cgMock.expects('connect').never(); + consumerGroup.scheduleTopicPartitionCheck(); + consumerGroup.closed = true; + clock.tick(30000); + + cgMock.verify(); + }); + + it('should still schedule check if topicPartitionChange errors', function () { + const cgMock = sandbox.mock(consumerGroup); + consumerGroup.isLeader = true; + cgMock + .expects('_checkTopicPartitionChange') + .once() + .yields(new Error('something bad')); + + cgMock.expects('commit').never(); + cgMock.expects('leaveGroup').never(); + cgMock.expects('connect').never(); + + consumerGroup.on('error', function () {}); + + consumerGroup.scheduleTopicPartitionCheck(); + sandbox.spy(consumerGroup, 'scheduleTopicPartitionCheck'); + + clock.tick(30000); + + cgMock.verify(); + sinon.assert.calledOnce(consumerGroup.scheduleTopicPartitionCheck); + }); + + it('should only have one schedule pending', function () { + const cgMock = sandbox.mock(consumerGroup); + consumerGroup.isLeader = true; + cgMock + .expects('_checkTopicPartitionChange') + .once() + .yields(null, true); + cgMock.expects('commit').never(); + cgMock + .expects('leaveGroup') + .once() + .yields(null); + cgMock.expects('connect').once(); + + consumerGroup.scheduleTopicPartitionCheck(); + consumerGroup.scheduleTopicPartitionCheck(); + + clock.tick(30000); + + cgMock.verify(); + }); + + it('should only schedule a check if consumer is a leader', function () { + const cgMock = sandbox.mock(consumerGroup); + + consumerGroup.isLeader = false; + + cgMock.expects('_checkTopicPartitionChange').never(); + cgMock.expects('leaveGroup').never(); + cgMock.expects('connect').never(); + cgMock.expects('commit').never(); + + consumerGroup.scheduleTopicPartitionCheck(); + clock.tick(30000); + cgMock.verify(); + }); + }); + + describe('#_checkTopicPartitionChange', function () { + it('should yield false when the topic/partition length are the same', function (done) { + sandbox.stub(consumerGroup.client, 'loadMetadataForTopics').yields(null, [ + 0, + { + metadata: { + aTopic: { + '0': {}, + '1': {} + }, + 'foo.bar.topic': { + '0': {}, + '1': {} + }, + existingTopic: { + '0': {}, + '1': {}, + '2': {} + } + } + } + ]); + + consumerGroup.topicPartitionLength = { + aTopic: 2, + 'foo.bar.topic': 2, + existingTopic: 3 + }; + + consumerGroup.topics = ['aTopic', 'existingTopic', 'foo.bar.topic']; + + consumerGroup._checkTopicPartitionChange(function (error, changed) { + sinon.assert.calledOnce(consumerGroup.client.loadMetadataForTopics); + should(changed).be.false; + done(error); + }); + }); + + it('should yield true when the topic/partition length are different', function (done) { + sandbox.stub(consumerGroup.client, 'loadMetadataForTopics').yields(null, [ + 0, + { + metadata: { + 'non.existant.topic': { + '0': {}, + '1': {} + }, + existingTopic: { + '0': {}, + '1': {}, + '2': {} + } + } + } + ]); + + consumerGroup.topicPartitionLength = { + 'non.existant.topic': 0, + existingTopic: 3 + }; + + consumerGroup.topics = ['existingTopic', 'non.existant.topic']; + + consumerGroup._checkTopicPartitionChange(function (error, changed) { + sinon.assert.calledOnce(consumerGroup.client.loadMetadataForTopics); + should(changed).be.true; + done(error); + }); + }); + }); + }); + + describe('Broker offline recovery', function () { + let sandbox = null; + let consumerGroup = null; + let fakeClient = null; + let ConsumerGroup = null; + let clock = null; + + fakeClient = function () { + return new EventEmitter(); + }; + + before(function () { + ConsumerGroup = proxyquire('../lib/consumerGroup', { + './kafkaClient': fakeClient + }); + + consumerGroup = new ConsumerGroup( + { + kafkaHost: 'gibberish', + connectOnReady: false + }, + 'TestTopic' + ); + }); + + beforeEach(function () { + sandbox = sinon.sandbox.create(); + + consumerGroup.client.refreshBrokerMetadata = sandbox.stub().yields(null); + clock = sandbox.useFakeTimers(); + }); + + afterEach(function () { + sandbox.restore(); + }); + + it('should refreshBrokerMetadata and connect when broker changes', function () { + sandbox.stub(consumerGroup, 'connect'); + sandbox.spy(consumerGroup, 'pause'); + + consumerGroup.ready = false; + consumerGroup.reconnectTimer = null; + consumerGroup.connecting = undefined; + + consumerGroup.client.emit('brokersChanged'); + clock.tick(50); + consumerGroup.client.emit('brokersChanged'); + clock.tick(100); + consumerGroup.client.emit('brokersChanged'); + clock.tick(200); + + sinon.assert.calledThrice(consumerGroup.pause); + sinon.assert.calledOnce(consumerGroup.client.refreshBrokerMetadata); + sinon.assert.calledOnce(consumerGroup.connect); + }); + + it('should not try to connect when broker changes and already connected', function () { + sandbox.stub(consumerGroup, 'connect'); + sandbox.spy(consumerGroup, 'pause'); + sandbox.stub(consumerGroup, 'fetch'); + + consumerGroup.ready = true; + consumerGroup.reconnectTimer = null; + consumerGroup.connecting = undefined; + + consumerGroup.client.emit('brokersChanged'); + clock.tick(200); + + sinon.assert.notCalled(consumerGroup.connect); + sinon.assert.calledOnce(consumerGroup.pause); + sinon.assert.calledOnce(consumerGroup.fetch); + sinon.assert.calledOnce(consumerGroup.client.refreshBrokerMetadata); + consumerGroup.paused.should.be.false; + }); + + it('should try to connect when broker changes and a reconnect is scheduled', function () { + let stub = sandbox.stub(consumerGroup, 'connect'); + sinon.stub(global, 'clearTimeout'); + + consumerGroup.ready = false; + consumerGroup.reconnectTimer = 1234; + consumerGroup.connecting = undefined; + + consumerGroup.client.emit('brokersChanged'); + clock.tick(200); + + should(consumerGroup.reconnectTimer).be.null; + sinon.assert.calledOnce(clearTimeout); + sinon.assert.calledOnce(stub); + global.clearTimeout.restore(); + }); + }); + + describe('Offset Out Of Range', function () { + const InvalidConsumerOffsetError = require('../lib/errors/InvalidConsumerOffsetError'); + + let ConsumerGroup = null; + let consumerGroup = null; + let sandbox = null; + let fakeClient = null; + + beforeEach(function () { + sandbox = sinon.sandbox.create(); + + fakeClient = sandbox.stub().returns(new EventEmitter()); + + ConsumerGroup = proxyquire('../lib/consumerGroup', { + './kafkaClient': fakeClient + }); + + consumerGroup = new ConsumerGroup( + { + connectOnReady: false, + sessionTimeout: 8000, + heartbeatInterval: 250, + retryMinTimeout: 250, + heartbeatTimeoutMs: 200 + }, + 'TestTopic' + ); + }); + + afterEach(function () { + sandbox.restore(); + }); + + it('should emit error on outOfRangeOffset event if outOfRangeOffset option is none', function (done) { + sandbox.spy(consumerGroup, 'pause'); + consumerGroup.on('error', function (error) { + error.should.be.an.instanceOf(InvalidConsumerOffsetError); + sinon.assert.calledOnce(consumerGroup.pause); + done(); + }); + + consumerGroup.options.outOfRangeOffset = 'none'; + consumerGroup.emit('offsetOutOfRange', { topic: 'test-topic', partition: '0' }); + }); + + it('should emit error if fetchOffset fails', function (done) { + const TOPIC_NAME = 'test-topic'; + + sandbox.stub(consumerGroup, 'resume'); + + consumerGroup.once('error', function (error) { + error.should.be.an.instanceOf(InvalidConsumerOffsetError); + sinon.assert.notCalled(consumerGroup.resume); + sinon.assert.notCalled(consumerGroup.setOffset); + done(); + }); + + consumerGroup.offset = { fetch: function () {} }; + sandbox.spy(consumerGroup, 'setOffset'); + sandbox.stub(consumerGroup.offset, 'fetch').yields(new Error('something went wrong')); + + consumerGroup.topicPayloads = [ + { + topic: TOPIC_NAME, + partition: '0', + offset: 1 + } + ]; + + consumerGroup.options.outOfRangeOffset = 'latest'; + consumerGroup.emit('offsetOutOfRange', { topic: TOPIC_NAME, partition: '0' }); + }); + + it('should set offset to latest if outOfRangeOffset is latest', function (done) { + const TOPIC_NAME = 'test-topic'; + const NEW_OFFSET = 657; + + sandbox.stub(consumerGroup, 'resume').callsFake(function () { + sinon.assert.calledOnce(consumerGroup.pause); + sinon.assert.calledWithExactly( + consumerGroup.offset.fetch, + [{ topic: TOPIC_NAME, partition: '0', time: -1 }], + sinon.match.func + ); + sinon.assert.calledWithExactly(consumerGroup.setOffset, TOPIC_NAME, '0', NEW_OFFSET); + consumerGroup.topicPayloads[0].offset.should.be.eql(NEW_OFFSET); + done(); + }); + + consumerGroup.offset = { fetch: function () {} }; + + sandbox.spy(consumerGroup, 'setOffset'); + sandbox.stub(consumerGroup, 'pause'); + + sandbox.stub(consumerGroup.offset, 'fetch').yields(null, { + 'test-topic': { + '0': [NEW_OFFSET] + } + }); + + consumerGroup.topicPayloads = [ + { + topic: TOPIC_NAME, + partition: '0', + offset: 1 + } + ]; + + consumerGroup.options.outOfRangeOffset = 'latest'; + consumerGroup.emit('offsetOutOfRange', { topic: TOPIC_NAME, partition: '0' }); + }); + + it('should set offset to earliest if outOfRangeOffset is earliest', function (done) { + const TOPIC_NAME = 'test-topic'; + const NEW_OFFSET = 500; + + sandbox.stub(consumerGroup, 'resume').callsFake(function () { + sinon.assert.calledWithExactly( + consumerGroup.offset.fetch, + [{ topic: TOPIC_NAME, partition: '0', time: -2 }], + sinon.match.func + ); + sinon.assert.calledWithExactly(consumerGroup.setOffset, TOPIC_NAME, '0', NEW_OFFSET); + consumerGroup.topicPayloads[0].offset.should.be.eql(NEW_OFFSET); + done(); + }); + + consumerGroup.offset = { fetch: function () {} }; + + sandbox.spy(consumerGroup, 'setOffset'); + + sandbox.stub(consumerGroup.offset, 'fetch').yields(null, { + 'test-topic': { + '0': [NEW_OFFSET] + } + }); + + consumerGroup.topicPayloads = [ + { + topic: TOPIC_NAME, + partition: '0', + offset: 1 + } + ]; + + consumerGroup.options.outOfRangeOffset = 'earliest'; + consumerGroup.emit('offsetOutOfRange', { topic: TOPIC_NAME, partition: '0' }); + }); + }); + + describe('#reconnectIfNeeded', function () { + let cg; + beforeEach(function () { + cg = { + connect: sinon.stub(), + fetch: sinon.stub() + }; + }); + + it('should do nothing if there is a pending connect', function () { + cg.connecting = true; + + ConsumerGroup.prototype.reconnectIfNeeded.call(cg); + + sinon.assert.notCalled(cg.connect); + sinon.assert.notCalled(cg.fetch); + should(cg.paused).be.false; + }); + + it('should call connect if not ready and not connecting', function () { + cg.connecting = false; + cg.ready = false; + + ConsumerGroup.prototype.reconnectIfNeeded.call(cg); + + sinon.assert.calledOnce(cg.connect); + sinon.assert.notCalled(cg.fetch); + should(cg.paused).be.false; + }); + + it('should call connect if not ready and not connecting and clear existing reconnectTimer', function () { + cg.connecting = false; + cg.ready = false; + + cg.reconnectTimer = {}; + + sinon.stub(global, 'clearTimeout'); + + ConsumerGroup.prototype.reconnectIfNeeded.call(cg); + + sinon.assert.calledOnce(cg.connect); + sinon.assert.calledOnce(clearTimeout); + sinon.assert.notCalled(cg.fetch); + should(cg.paused).be.false; + should(cg.reconnectTimer).be.null; + global.clearTimeout.restore(); + }); + + it('should call fetch if ready and not connecting', function () { + cg.connecting = false; + cg.ready = true; + + ConsumerGroup.prototype.reconnectIfNeeded.call(cg); + + sinon.assert.notCalled(cg.connect); + sinon.assert.calledOnce(cg.fetch); + should(cg.paused).be.false; + }); + }); + + describe('#clearPendingFetches', function () { + it('should set waiting to false and clear the callback queue', function () { + const pendingSocket = new FakeSocket(); + pendingSocket.waiting = true; + const longPollingBrokers = { + '1': new BrokerWrapper(pendingSocket), + '2': new BrokerWrapper(new FakeSocket()) + }; + const fakeClient = { + getBrokers: sinon.stub().returns(longPollingBrokers), + clearCallbackQueue: sinon.stub() + }; + + ConsumerGroup.prototype.clearPendingFetches.call({ + client: fakeClient + }); + + sinon.assert.calledWithExactly(fakeClient.getBrokers, true); + sinon.assert.calledWithExactly(fakeClient.clearCallbackQueue, pendingSocket); + pendingSocket.waiting.should.be.false; + }); + }); + + describe('#close', function () { + let sandbox, consumerGroup; + + beforeEach(function () { + sandbox = sinon.sandbox.create(); + consumerGroup = new ConsumerGroup( + { + connectOnReady: false, + sessionTimeout: 8000, + heartbeatInterval: 250, + retryMinTimeout: 250, + heartbeatTimeoutMs: 200 + }, + 'TestTopic' + ); + }); + + afterEach(function (done) { + consumerGroup.close(function () { + sandbox.restore(); + done(); + }); + }); + + it('make an attempt to leave the group but do not error out when it fails', function (done) { + const NotCoordinatorForGroup = require('../lib/errors/NotCoordinatorForGroupError'); + sandbox.stub(consumerGroup, 'leaveGroup').yields(new NotCoordinatorForGroup()); + consumerGroup.connect(); + consumerGroup.once('connect', function () { + consumerGroup.close(done); + }); + }); + + it('should not throw an exception when closing immediately after an UnknownMemberId error', function (done) { + const UnknownMemberId = require('../lib/errors/UnknownMemberIdError'); + sandbox.stub(consumerGroup.client, 'sendHeartbeatRequest').yields(new UnknownMemberId('test')); + + should.doesNotThrow(function () { + consumerGroup.connect(); + consumerGroup.once('connect', function () { + consumerGroup.close(done); + }); + }); + }); + }); + + describe('Long running fetches', function () { + let consumerGroup; + + beforeEach(function (done) { + consumerGroup = new ConsumerGroup( + { + groupId: 'longFetchSimulation' + }, + 'TestTopic' + ); + consumerGroup.once('connect', done); + }); + + afterEach(function (done) { + consumerGroup.close(done); + }); + + it('should not throw out of bounds', function (done) { + should.doesNotThrow(function () { + consumerGroup.pause(); + consumerGroup.client.correlationId = 2147483647; + consumerGroup.resume(); + setImmediate(done); + }); + }); + }); + + describe('Sending Heartbeats', function () { + var consumerGroup, sandbox; + + beforeEach(function () { + consumerGroup = new ConsumerGroup( + { + connectOnReady: false + }, + 'TestTopic' + ); + + sandbox = sinon.sandbox.create(); + sandbox.stub(consumerGroup, 'sendHeartbeat').returns({ + verifyResolved: sandbox.stub().returns(true) + }); + sandbox.useFakeTimers(); + }); + + afterEach(function (done) { + sandbox.restore(); + consumerGroup.close(done); + }); + + it('throws exception if consumerGroup is not ready', function () { + should.throws(function () { + consumerGroup.startHeartbeats(); + }); + }); + + it('should not continue to send heartbeats if last one never resolved', function () { + consumerGroup.ready = true; + sinon.assert.notCalled(consumerGroup.sendHeartbeat); + + consumerGroup.sendHeartbeat.restore(); + + const verifyResolvedStub = sandbox.stub().returns(false); + sandbox.stub(consumerGroup, 'sendHeartbeat').returns({ + verifyResolved: verifyResolvedStub + }); + + consumerGroup.options.heartbeatInterval = 3000; + + consumerGroup.startHeartbeats(); + sinon.assert.calledOnce(consumerGroup.sendHeartbeat); + + sandbox.clock.tick(2000); + sinon.assert.calledOnce(consumerGroup.sendHeartbeat); + + sandbox.clock.tick(1000); + sinon.assert.calledOnce(consumerGroup.sendHeartbeat); + + sandbox.clock.tick(1000); + sinon.assert.calledOnce(consumerGroup.sendHeartbeat); + }); + + it('should use heartbeatInterval passed into options', function () { + consumerGroup.ready = true; + sinon.assert.notCalled(consumerGroup.sendHeartbeat); + consumerGroup.options.heartbeatInterval = 3000; + + consumerGroup.startHeartbeats(); + + sinon.assert.calledOnce(consumerGroup.sendHeartbeat); + sandbox.clock.tick(2000); + sinon.assert.calledOnce(consumerGroup.sendHeartbeat); + sandbox.clock.tick(1000); + sinon.assert.calledTwice(consumerGroup.sendHeartbeat); + sandbox.clock.tick(3000); + sinon.assert.calledThrice(consumerGroup.sendHeartbeat); + }); + }); + + describe('#handleSyncGroup', function () { + var consumerGroup, sandbox; + beforeEach(function () { + consumerGroup = new ConsumerGroup( + { + connectOnReady: false + }, + 'TestTopic' + ); + sandbox = sinon.sandbox.create(); + }); + + afterEach(function (done) { + sandbox.restore(); + consumerGroup.close(done); + }); + + describe('options.fromOffset is "none"', function () { + it('should yield error when there is not saved offsets', function (done) { + consumerGroup.options.fromOffset = 'none'; + const syncGroupResponse = { + partitions: { + TestTopic: [0, 2, 3, 4] + } + }; + + const fetchOffsetResponse = { + TestTopic: { + 0: 10, + 2: -1, + 3: -1, + 4: -1 + } + }; + + sandbox.stub(consumerGroup, 'fetchOffset').yields(null, fetchOffsetResponse); + sandbox.stub(consumerGroup, 'saveDefaultOffsets').yields(null); + consumerGroup.handleSyncGroup(syncGroupResponse, function (error, ownsPartitions) { + should(ownsPartitions).be.undefined; + error.should.be.a.Error; + done(); + }); + }); + }); + + describe('options.fromOffset is "latest"', function () { + it('should not fetch latestOffset if all offsets have saved previously', function (done) { + consumerGroup.options.fromOffset = 'latest'; + + const syncGroupResponse = { + partitions: { + TestTopic: [0, 2, 3, 4] + } + }; + + const fetchOffsetResponse = { + TestTopic: { + 0: 10, + 2: 0, + 3: 9, + 4: 6 + } + }; + + sandbox.stub(consumerGroup, 'fetchOffset').yields(null, fetchOffsetResponse); + sandbox.stub(consumerGroup, 'saveDefaultOffsets').yields(null); + + consumerGroup.handleSyncGroup(syncGroupResponse, function (error, ownsPartitions) { + ownsPartitions.should.be.true; + sinon.assert.calledWith(consumerGroup.fetchOffset, syncGroupResponse.partitions); + sinon.assert.notCalled(consumerGroup.saveDefaultOffsets); + + const topicPayloads = _(consumerGroup.topicPayloads); + + topicPayloads.find({ topic: 'TestTopic', partition: 0 }).offset.should.be.eql(10); + topicPayloads.find({ topic: 'TestTopic', partition: 2 }).offset.should.be.eql(0); + topicPayloads.find({ topic: 'TestTopic', partition: 3 }).offset.should.be.eql(9); + topicPayloads.find({ topic: 'TestTopic', partition: 4 }).offset.should.be.eql(6); + done(error); + }); + }); + + it('should use latestOffset if offsets have never been saved', function (done) { + consumerGroup.options.fromOffset = 'latest'; + + const syncGroupResponse = { + partitions: { + TestTopic: [0, 2, 3, 4] + } + }; + + const fetchOffsetResponse = { + TestTopic: { + 0: 10, + 2: -1, + 3: -1, + 4: -1 + } + }; + + const defaultOffsets = { + TestTopic: { + 0: 10, + 2: 3, + 4: 5000 + } + }; + + consumerGroup.defaultOffsets = defaultOffsets; + sandbox.stub(consumerGroup, 'fetchOffset').yields(null, fetchOffsetResponse); + sandbox.stub(consumerGroup, 'saveDefaultOffsets').yields(null); + + consumerGroup.handleSyncGroup(syncGroupResponse, function (error, ownsPartitions) { + ownsPartitions.should.be.true; + sinon.assert.calledWith(consumerGroup.fetchOffset, syncGroupResponse.partitions); + sinon.assert.calledOnce(consumerGroup.saveDefaultOffsets); + + const topicPayloads = _(consumerGroup.topicPayloads); + + topicPayloads.find({ topic: 'TestTopic', partition: 3 }).offset.should.be.eql(0); + topicPayloads.find({ topic: 'TestTopic', partition: 2 }).offset.should.be.eql(3); + topicPayloads.find({ topic: 'TestTopic', partition: 0 }).offset.should.be.eql(10); + topicPayloads.find({ topic: 'TestTopic', partition: 4 }).offset.should.be.eql(5000); + done(error); + }); + }); + }); + + it('should yield false when there are no partitions owned', function (done) { + consumerGroup.handleSyncGroup( + { + partitions: {} + }, + function (error, ownsPartitions) { + ownsPartitions.should.be.false; + done(error); + } + ); + }); + }); + + describe('KafkaClient support', function () { + it('should throw error if migration option is used with KafkaClient', function () { + should.throws(function () { + // eslint-disable-next-line no-new + new ConsumerGroup( + { + kafkaHost: 'localhost:9092', + migrateHLC: true, + connectOnReady: false, + autoConnect: false + }, + 'TestTopic' + ); + }); + }); + }); + + function sendRandomByteMessage (bytes, topic, done) { + const crypto = require('crypto'); + const buffer = crypto.randomBytes(bytes); + + sendMessage(buffer, topic, done); + + return buffer; + } + + describe('fetchMaxBytes', function () { + let topic, consumerGroup; + beforeEach(function (done) { + topic = uuid.v4(); + sendRandomByteMessage(2048, topic, done); + }); + + afterEach(function (done) { + consumerGroup.close(done); + }); + + it('should throw an error when message exceeds maximum fetch size', function (done) { + consumerGroup = new ConsumerGroup( + { + kafkaHost: '127.0.0.1:9092', + groupId: uuid.v4(), + fetchMaxBytes: 1024, + fromOffset: 'earliest' + }, + topic + ); + + consumerGroup.once('error', function (error) { + error.should.be.an.instanceOf(require('../lib/errors/MessageSizeTooLargeError')); + done(); + }); + + consumerGroup.once('message', function (message) { + done(new Error('should not receive a message')); + }); + }); + }); + + describe('#scheduleReconnect', function () { + var consumerGroup, sandbox; + + beforeEach(function () { + consumerGroup = new ConsumerGroup( + { + connectOnReady: false + }, + 'TestTopic' + ); + + sandbox = sinon.sandbox.create(); + sandbox.stub(consumerGroup, 'connect'); + }); + + afterEach(function (done) { + sandbox.restore(); + consumerGroup.close(done); + }); + + it('should throw an exception if passed in a empty timeout', function () { + should.throws(function () { + consumerGroup.scheduleReconnect(0); + }); + + should.throws(function () { + consumerGroup.scheduleReconnect(null); + }); + + should.throws(function () { + consumerGroup.scheduleReconnect(undefined); + }); + }); + + it('should call connect method when scheduled timer is up', function () { + sandbox.useFakeTimers(); + should(consumerGroup.reconnectTimer).be.undefined; + consumerGroup.scheduleReconnect(1000); + should(consumerGroup.reconnectTimer).not.be.empty; + sandbox.clock.tick(1000); + should(consumerGroup.reconnectTimer).be.null; + sinon.assert.calledOnce(consumerGroup.connect); + }); + + it('should only allow one connect to be scheduled at a time', function () { + sandbox.useFakeTimers(); + consumerGroup.scheduleReconnect(2000); + sandbox.clock.tick(1000); + consumerGroup.scheduleReconnect(2000); + consumerGroup.scheduleReconnect(2000); + consumerGroup.scheduleReconnect(2000); + sandbox.clock.tick(2000); + should(consumerGroup.reconnectTimer).be.null; + sinon.assert.calledOnce(consumerGroup.connect); + + sandbox.clock.tick(10000); + sinon.assert.calledOnce(consumerGroup.connect); + }); + }); + + it('should not commit when autoCommit is disabled', function (done) { + const topic = uuid.v4(); + const groupId = uuid.v4(); + + const messages = _.times(5, function () { + return uuid.v4(); + }); + + const createTopic = require('../docker/createTopic'); + + function addMessages (done) { + const Producer = require('../lib/producer'); + const KafkaClient = require('../lib/kafkaClient'); + + const client = new KafkaClient(); + const producer = new Producer(client); + + async.series( + [ + function (callback) { + createTopic(topic, 1, 1).then(function () { + callback(null); + }, callback); + }, + function (callback) { + if (producer.ready) { + return callback(); + } + producer.once('ready', callback); + }, + function (callback) { + producer.send([{ topic: topic, messages: messages }], callback); + }, + function (callback) { + producer.close(callback); + } + ], + done + ); + } + + function confirmMessages (done) { + const left = messages.slice(); + const consumerGroup = new ConsumerGroup( + { + fromOffset: 'earliest', + groupId: groupId, + sessionTimeout: 8000, + autoCommit: false + }, + topic + ); + + async.series( + [ + function (callback) { + consumerGroup.on('message', function (data) { + _.pull(left, data.value); + + if (left.length === 0) { + callback(); + } + }); + }, + function (callback) { + consumerGroup.close(callback); + } + ], + done + ); + } + + async.series([addMessages, confirmMessages, confirmMessages], done); + }); + + describe('#addTopics', function () { + let topic, newTopic, testMessage, consumerGroup; + + before('create the topics', done => { + topic = uuid.v4(); + newTopic = uuid.v4(); + testMessage = uuid.v4(); + consumerGroup = new ConsumerGroup( + { + groupId: uuid.v4() + }, + topic + ); + consumerGroup.once('connect', () => { + consumerGroup.client.createTopics([topic, newTopic], done); + }); + }); + + after('close consumer group', done => { + consumerGroup.close(done); + }); + + it('should fetch messages from the topic added', done => { + let messages = []; + consumerGroup.on('message', message => { + messages.push(message); + if (messages.length === 2) { + messages.should.containDeep([ + { + topic: topic, + value: testMessage + }, + { + topic: newTopic, + value: testMessage + } + ]); + done(); + } + }); + consumerGroup.addTopics([newTopic], (error, result) => { + should(error).be.null; + result.should.be.eql(`Add Topics ${newTopic} Successfully`); + consumerGroup.once('connect', () => { + sendMessage(testMessage, topic, () => {}); + sendMessage(testMessage, newTopic, () => {}); + }); + }); + }); + }); + + describe('#fetch', function () { + let consumerGroup; + + afterEach(function (done) { + consumerGroup.close(done); + }); + + it('should reset fetch pending state if fetch request fails', function () { + const topic = uuid.v4(); + consumerGroup = new ConsumerGroup( + { + connectOnReady: false, + groupId: uuid.v4(), + autoCommit: false + }, + [topic] + ); + + consumerGroup.ready = true; + consumerGroup.paused = false; + consumerGroup._isFetchPending = false; + + const clientMock = sinon.mock(consumerGroup.client); + + clientMock + .expects('sendFetchRequest') + .once() + .yields(new BrokerNotAvailableError('Test Error')); + + const cgMock = sinon.mock(consumerGroup); + cgMock.expects('_resetFetchState').once(); + + consumerGroup.fetch(); + clientMock.verify(); + cgMock.verify(); + }); + + it('should not reset fetch pending state if fetch request was successful', function () { + const topic = uuid.v4(); + consumerGroup = new ConsumerGroup( + { + connectOnReady: false, + groupId: uuid.v4(), + autoCommit: false + }, + [topic] + ); + + consumerGroup.ready = true; + consumerGroup.paused = false; + consumerGroup._isFetchPending = false; + + const clientMock = sinon.mock(consumerGroup.client); + + clientMock + .expects('sendFetchRequest') + .once() + .yields(null); + + const cgMock = sinon.mock(consumerGroup); + cgMock.expects('_resetFetchState').never(); + + consumerGroup.fetch(); + clientMock.verify(); + cgMock.verify(); + }); + }); + + describe('#removeTopics', function () { + let topic, newTopic, testMessage, consumerGroup; + + before('create the topics', done => { + topic = uuid.v4(); + newTopic = uuid.v4(); + testMessage = uuid.v4(); + consumerGroup = new ConsumerGroup( + { + groupId: uuid.v4(), + autoCommit: false + }, + [topic, newTopic] + ); + consumerGroup.once('connect', () => { + consumerGroup.client.createTopics([topic, newTopic], done); + }); + }); + + after('close consumer group', done => { + consumerGroup.close(done); + }); + + it('should not fetch messages from the topic removed', done => { + let messages = []; + consumerGroup.on('message', message => { + messages.push(message); + if (messages.length === 1) { + messages.should.containDeep([ + { + topic: topic, + value: testMessage + } + ]); + done(); + } + }); + consumerGroup.removeTopics([newTopic], (error, result) => { + should(error).be.null; + result.should.be.eql(`Remove Topics ${newTopic} Successfully`); + consumerGroup.once('connect', () => { + sendMessage(testMessage, newTopic, () => {}); + sendMessage(testMessage, topic, () => {}); + }); + }); + }); + }); +}); diff --git a/node_modules/kafka-node/test/test.consumerGroupHeartbeat.js b/node_modules/kafka-node/test/test.consumerGroupHeartbeat.js new file mode 100644 index 0000000..e0c6c2d --- /dev/null +++ b/node_modules/kafka-node/test/test.consumerGroupHeartbeat.js @@ -0,0 +1,72 @@ +'use strict'; + +const sinon = require('sinon'); +const should = require('should'); +const Heartbeat = require('../lib/consumerGroupHeartbeat'); +const HeartbeatTimeout = require('../lib/errors/HeartbeatTimeoutError'); + +describe('Consumer Group Heartbeat', function () { + let sandbox; + + beforeEach(function () { + sandbox = sinon.sandbox.create(); + }); + + afterEach(function () { + sandbox.restore(); + }); + + it('should call heartbeat handler if heartbeat yields error', function (done) { + const mockClient = { + sendHeartbeatRequest: sandbox.stub().yieldsAsync(new Error('busted')) + }; + + const heartbeat = new Heartbeat(mockClient, function (error) { + error.message.should.eql('busted'); + sinon.assert.calledWithExactly(mockClient.sendHeartbeatRequest, 'groupId', 1, 'fake-member-id', sinon.match.func); + heartbeat.pending.should.be.false; + setImmediate(done); + }); + + heartbeat.pending.should.be.true; + heartbeat.send('groupId', 1, 'fake-member-id'); + setImmediate(() => heartbeat.verifyResolved().should.be.true); + }); + + it('should call heartbeat handler if heartbeat yields null', function (done) { + const mockClient = { + sendHeartbeatRequest: sandbox.stub().yieldsAsync(null) + }; + + const heartbeat = new Heartbeat(mockClient, function (error) { + should(error).be.null; + sinon.assert.calledWithExactly(mockClient.sendHeartbeatRequest, 'groupId', 1, 'fake-member-id', sinon.match.func); + heartbeat.pending.should.be.false; + setImmediate(done); + }); + + heartbeat.pending.should.be.true; + heartbeat.send('groupId', 1, 'fake-member-id'); + setImmediate(() => heartbeat.verifyResolved().should.be.true); + }); + + it('should call heartbeat handler with instance of TimeoutError if heartbeat timed out', function (done) { + const mockClient = { + sendHeartbeatRequest: sandbox.stub() + }; + + const heartbeat = new Heartbeat(mockClient, function (error) { + error.should.be.an.instanceOf(HeartbeatTimeout); + sinon.assert.calledWithExactly(mockClient.sendHeartbeatRequest, 'groupId', 1, 'fake-member-id', sinon.match.func); + heartbeat.pending.should.be.false; + heartbeat.canceled.should.be.true; + setImmediate(done); + }); + + heartbeat.pending.should.be.true; + heartbeat.send('groupId', 1, 'fake-member-id'); + setImmediate(function () { + heartbeat.verifyResolved().should.be.false; + }); + }); +}); diff --git a/node_modules/kafka-node/test/test.consumerGroupRecovery.js b/node_modules/kafka-node/test/test.consumerGroupRecovery.js new file mode 100644 index 0000000..ed57f60 --- /dev/null +++ b/node_modules/kafka-node/test/test.consumerGroupRecovery.js @@ -0,0 +1,205 @@ +'use strict'; + +const should = require('should'); +const _ = require('lodash'); +const sinon = require('sinon'); +const ConsumerGroupRecovery = require('../lib/consumerGroupRecovery'); +const GroupCoordinatorNotAvailable = require('../lib/errors/GroupCoordinatorNotAvailableError'); +const GroupLoadInProgress = require('../lib/errors/GroupLoadInProgressError'); +const HeartbeatTimeout = require('../lib/errors/HeartbeatTimeoutError'); +const TimeoutError = require('../lib/errors/TimeoutError'); +const BrokerNotAvailableError = require('../lib/errors').BrokerNotAvailableError; +const EventEmitter = require('events'); + +describe('ConsumerGroupRecovery', function () { + var consumerGroupRecovery, fakeConsumerGroup; + + beforeEach(function () { + fakeConsumerGroup = new EventEmitter(); + fakeConsumerGroup.client = new EventEmitter(); + fakeConsumerGroup.scheduleReconnect = () => { + throw new Error('should be stubbed!'); + }; + Object.assign(fakeConsumerGroup, { + stopHeartbeats: sinon.stub(), + options: { + retries: 10, + retryFactor: 1.8, + retryMinTimeout: 1000 + } + }); + consumerGroupRecovery = new ConsumerGroupRecovery(fakeConsumerGroup); + }); + + describe('#tryToRecoverFrom', function () { + it('should emit error on the client when calling trying to recover from a unknown error', function (done) { + var testError = new Error('My test error'); + + fakeConsumerGroup.once('error', function (error) { + error.should.be.eql(testError); + done(); + }); + + consumerGroupRecovery.tryToRecoverFrom(testError, 'test'); + + sinon.assert.calledOnce(fakeConsumerGroup.stopHeartbeats); + fakeConsumerGroup.ready.should.be.false; + consumerGroupRecovery.lastError.should.be.eql(testError); + }); + + it('should try to recover from a BrokerNotAvailableError', function () { + const brokerNotAvailableError = new BrokerNotAvailableError('test error'); + + fakeConsumerGroup.client.coordinatorId = 1234; + + fakeConsumerGroup.once('error', function (error) { + error.should.not.be.eql(brokerNotAvailableError); + }); + + sinon.stub(fakeConsumerGroup, 'scheduleReconnect'); + + consumerGroupRecovery.tryToRecoverFrom(brokerNotAvailableError, 'test'); + + sinon.assert.calledOnce(fakeConsumerGroup.stopHeartbeats); + fakeConsumerGroup.ready.should.be.false; + consumerGroupRecovery.lastError.should.be.eql(brokerNotAvailableError); + + sinon.assert.calledOnce(fakeConsumerGroup.scheduleReconnect); + should(fakeConsumerGroup.client.coordinatorId).be.undefined; + }); + + it('should try to recover from a temporary network error', function () { + const fakeNetworkError = new Error('read ETIMEDOUT'); + fakeNetworkError.code = fakeNetworkError.errno = 'ETIMEDOUT'; + + fakeConsumerGroup.once('error', function (error) { + error.should.not.be.eql(fakeNetworkError); + }); + + sinon.stub(fakeConsumerGroup, 'scheduleReconnect'); + + consumerGroupRecovery.tryToRecoverFrom(fakeNetworkError, 'test'); + + sinon.assert.calledOnce(fakeConsumerGroup.stopHeartbeats); + fakeConsumerGroup.ready.should.be.false; + consumerGroupRecovery.lastError.should.be.eql(fakeNetworkError); + + sinon.assert.calledOnce(fakeConsumerGroup.scheduleReconnect); + }); + + it('should try to recover from a request timeout', function () { + const fakeNetworkError = new TimeoutError('request timeout'); + + fakeConsumerGroup.once('error', function (error) { + error.should.not.be.eql(fakeNetworkError); + }); + + sinon.stub(fakeConsumerGroup, 'scheduleReconnect'); + + consumerGroupRecovery.tryToRecoverFrom(fakeNetworkError, 'test'); + + sinon.assert.calledOnce(fakeConsumerGroup.stopHeartbeats); + fakeConsumerGroup.ready.should.be.false; + consumerGroupRecovery.lastError.should.be.eql(fakeNetworkError); + + sinon.assert.calledOnce(fakeConsumerGroup.scheduleReconnect); + }); + + it('should try to recover from a HeartbeatTimeout', function () { + const heartbeatTimeout = new HeartbeatTimeout('test error'); + + fakeConsumerGroup.once('error', function (error) { + error.should.not.be.eql(heartbeatTimeout); + }); + + sinon.stub(fakeConsumerGroup, 'scheduleReconnect'); + + consumerGroupRecovery.tryToRecoverFrom(heartbeatTimeout, 'test'); + + sinon.assert.calledOnce(fakeConsumerGroup.stopHeartbeats); + fakeConsumerGroup.ready.should.be.false; + consumerGroupRecovery.lastError.should.be.eql(heartbeatTimeout); + + sinon.assert.calledOnce(fakeConsumerGroup.scheduleReconnect); + }); + }); + + describe('#getRetryTimeout', function () { + it('should reset backoff timeouts when calling with different error', function () { + var error = new GroupCoordinatorNotAvailable(); + error.errorCode = 15; + + var differentError = new GroupLoadInProgress(); + differentError.errorCode = 14; + + consumerGroupRecovery.options.retries = 5; + + var results = _.times(3, function () { + var ret = consumerGroupRecovery.getRetryTimeout(error); + consumerGroupRecovery.lastError = error; + return ret; + }); + + consumerGroupRecovery._timeouts.should.have.length(5); + + results.should.be.length(3); + results.forEach(function (result, index) { + result.should.eql(consumerGroupRecovery._timeouts[index]); + }); + + results = _.times(5, function () { + var ret = consumerGroupRecovery.getRetryTimeout(differentError); + consumerGroupRecovery.lastError = differentError; + return ret; + }); + + consumerGroupRecovery._timeouts.should.have.length(5); + + results.should.be.length(5); + results.forEach(function (result, index) { + result.should.eql(consumerGroupRecovery._timeouts[index]); + }); + }); + + it('should return backoff timeouts when calling same error', function () { + var error = new GroupCoordinatorNotAvailable(); + error.errorCode = 15; + + consumerGroupRecovery.options.retries = 5; + + var results = _.times(6, function () { + var ret = consumerGroupRecovery.getRetryTimeout(error); + consumerGroupRecovery.lastError = error; + return ret; + }); + + consumerGroupRecovery._timeouts.should.have.length(5); + results.pop().should.be.false; + results.forEach(function (result, index) { + result.should.eql(consumerGroupRecovery._timeouts[index]); + }); + }); + + it('should initalize timeout array when invoked', function () { + var error = new GroupCoordinatorNotAvailable(); + error.errorCode = 15; + + should(consumerGroupRecovery._timeouts).be.undefined; + + var retryTime = consumerGroupRecovery.getRetryTimeout(error); + + consumerGroupRecovery._timeouts.should.have.length(10); + retryTime.should.be.eql(consumerGroupRecovery._timeouts[0]); + }); + + it('should throw if empty arguments', function () { + should.throws(function () { + consumerGroupRecovery.getRetryTimeout(); + }); + + should.throws(function () { + consumerGroupRecovery.getRetryTimeout(null); + }); + }); + }); +}); diff --git a/node_modules/kafka-node/test/test.consumerGroupStream.js b/node_modules/kafka-node/test/test.consumerGroupStream.js new file mode 100644 index 0000000..a949f13 --- /dev/null +++ b/node_modules/kafka-node/test/test.consumerGroupStream.js @@ -0,0 +1,227 @@ +'use strict'; + +const uuid = require('uuid'); +const createTopic = require('../docker/createTopic'); +const sendMessage = require('./helpers/sendMessage'); +const async = require('async'); +const _ = require('lodash'); +const sinon = require('sinon'); +const proxyquire = require('proxyquire'); +const EventEmitter = require('events'); + +const ConsumerGroupStream = require('../lib/consumerGroupStream'); + +function createConsumerGroupStream (topic, overrides) { + const consumerOptions = _.defaultsDeep(overrides || {}, { + kafkaHost: '127.0.0.1:9092', + groupId: 'ExampleTestGroup', + sessionTimeout: 15000, + protocol: ['roundrobin'], + asyncPush: false, + id: 'consumer1', + autoCommit: false, + fromOffset: 'earliest' + }); + return new ConsumerGroupStream(consumerOptions, topic); +} + +describe('ConsumerGroupStream', function () { + let topic, groupId, sandbox; + + context('autoCommit', function () { + let consumerGroupStream, ConsumerGroupStream, fakeConsumerGroup; + + before(function () { + sandbox = sinon.sandbox.create({ + useFakeTimers: true + }); + fakeConsumerGroup = new EventEmitter(); + _.set(fakeConsumerGroup, 'options.autoCommitIntervalMs', 500); + ConsumerGroupStream = proxyquire('../lib/consumerGroupStream', { + './consumerGroup': function () { + return fakeConsumerGroup; + } + }); + }); + + beforeEach(function () { + consumerGroupStream = new ConsumerGroupStream({}); + sandbox.reset(); + }); + + after(function () { + sandbox.restore(); + }); + + it('should continue to work work after force true commit', function (done) { + fakeConsumerGroup.sendOffsetCommitRequest = sinon.stub().yieldsAsync(null); + async.series( + [ + function (callback) { + consumerGroupStream.commit( + { + topic: 'MyTestTopic', + partition: 0, + offset: 1 + }, + true, + callback + ); + }, + function (callback) { + consumerGroupStream.commit( + { + topic: 'MyTestTopic', + partition: 0, + offset: 5 + }, + false, + callback + ); + } + ], + function (error) { + sinon.assert.calledTwice(fakeConsumerGroup.sendOffsetCommitRequest); + sinon.assert.calledWith(fakeConsumerGroup.sendOffsetCommitRequest, [ + { + metadata: 'm', + offset: 2, + partition: 0, + topic: 'MyTestTopic' + } + ]); + + sinon.assert.calledWith(fakeConsumerGroup.sendOffsetCommitRequest, [ + { + metadata: 'm', + offset: 6, + partition: 0, + topic: 'MyTestTopic' + } + ]); + + consumerGroupStream.committing.should.be.true; + sandbox.clock.tick(500); + consumerGroupStream.committing.should.be.false; + done(error); + } + ); + }); + }); + + context('with topic', function () { + beforeEach(function () { + topic = uuid.v4(); + groupId = uuid.v4(); + return createTopic(topic, 1, 1); + }); + + describe('Auto Commit', function () { + it('should resume at the next offset after new stream starts', function (done) { + let messages; + let consumerGroupStream; + let lastReadOffset; + async.series( + [ + function (callback) { + messages = _.times(3, uuid.v4); + sendMessage(messages, topic, callback); + }, + + function (callback) { + callback = _.once(callback); + const messagesToRead = _.clone(messages); + consumerGroupStream = createConsumerGroupStream(topic, { autoCommit: true, groupId: groupId }); + consumerGroupStream.on('data', function (message) { + _.pull(messagesToRead, message.value, message.offset); + if (messagesToRead.length === 0) { + lastReadOffset = message.offset; + callback(null); + } + }); + }, + + function (callback) { + setImmediate(function () { + consumerGroupStream.close(callback); + consumerGroupStream = null; + }); + }, + + function (callback) { + setTimeout(callback, 100); + }, + + function (callback) { + sendMessage([uuid.v4()], topic, callback); + }, + + function (callback) { + consumerGroupStream = createConsumerGroupStream(topic, { autoCommit: true, groupId: groupId }); + consumerGroupStream.on('readable', function () { + const message = consumerGroupStream.read(); + message.offset.should.be.equal(lastReadOffset + 1); + consumerGroupStream.close(callback); + }); + } + ], + done + ); + }); + }); + + describe('#close', function () { + it('should not call consumerGroup with force option', function (done) { + const consumerGroupStream = createConsumerGroupStream(topic); + + const closeSpy = sinon.spy(consumerGroupStream.consumerGroup, 'close'); + + consumerGroupStream.close(function () { + sinon.assert.calledOnce(closeSpy); + sinon.assert.calledWithExactly(closeSpy, false, sinon.match.func); + done(); + }); + }); + + it('autoCommit false should close the consumer without committing offsets', function (done) { + const messages = _.times(3, uuid.v4); + let consumerGroupStream; + + async.series( + [ + function (callback) { + sendMessage(messages, topic, callback); + }, + function (callback) { + const messagesToRead = _.clone(messages); + consumerGroupStream = createConsumerGroupStream(topic, { groupId: groupId }); + consumerGroupStream.on('data', function (message) { + _.pull(messagesToRead, message.value); + if (messagesToRead.length === 0) { + callback(null); + } + }); + }, + function (callback) { + consumerGroupStream.close(callback); + }, + function (callback) { + const messagesToRead = _.clone(messages); + consumerGroupStream = createConsumerGroupStream(topic, { groupId: groupId }); + consumerGroupStream.on('data', function (message) { + _.pull(messagesToRead, message.value); + if (messagesToRead.length === 0) { + callback(null); + } + }); + }, + function (callback) { + consumerGroupStream.close(callback); + } + ], + done + ); + }); + }); + }); +}); diff --git a/node_modules/kafka-node/test/test.consumerStream.js b/node_modules/kafka-node/test/test.consumerStream.js new file mode 100644 index 0000000..300f2e4 --- /dev/null +++ b/node_modules/kafka-node/test/test.consumerStream.js @@ -0,0 +1,200 @@ +'use strict'; +var should = require('should'); + +var through2 = require('through2'); + +var libPath = process.env['KAFKA_COV'] ? '../lib-cov/' : '../lib/'; +var ConsumerStream = require(libPath + 'consumerStream'); +var Producer = require(libPath + 'producer'); +var Client = require(libPath + 'kafkaClient'); +var EventCounter = require('./helpers/EventCounter'); + +const TOPIC_POSTFIX = '_test_' + Date.now(); +const EXISTS_TOPIC_1 = '_exists_1' + TOPIC_POSTFIX; +const APPEND_TOPIC_1 = '_append_1' + TOPIC_POSTFIX; +const COMMIT_STREAM_TOPIC_1 = '_commit_stream_1' + TOPIC_POSTFIX; +const COMMIT_STREAM_TOPIC_2 = '_commit_stream_2' + TOPIC_POSTFIX; +const COMMIT_STREAM_TOPIC_3 = '_commit_stream_3' + TOPIC_POSTFIX; + +function createTopicAndProduceMessages (producer, topic, numberOfMessages, done) { + if (!done) { + done = function () {}; + } + producer.createTopics([topic], function () { + var messages = []; + for (var i = 1; i <= numberOfMessages; i++) { + messages.push('stream message ' + i); + } + producer.send([{ topic: topic, messages }], done); + }); +} + +describe('ConsumerStream', function () { + it("should emit both a 'message' and a 'data' event for each message", function (done) { + var client = new Client(); + var producer = new Producer(client); + producer.once('ready', function () { + createTopicAndProduceMessages(producer, EXISTS_TOPIC_1, 100, function () { + var topics = [{ topic: EXISTS_TOPIC_1 }]; + // Here we set fetchMaxBytes to ensure that we're testing running + // multiple fetches as the default 1024 * 1024 makes a single fetch. + var options = { autoCommit: false, groupId: '_groupId_1_test', fetchMaxBytes: 512 }; + var consumer = new ConsumerStream(client, topics, options); + var eventCounter = new EventCounter(); + consumer.on('message', eventCounter.createEventCounter('message')); + consumer.on('data', eventCounter.createEventCounter('data')); + var incrementPipeCount = eventCounter.createEventCounter('pipe', 100, function () { + should.exist(eventCounter.events.message.events); + var events = eventCounter.events; + should.exist(events.message); + events.message.events.length.should.equal(100); + events.data.events.length.should.equal(100); + events.pipe.events.length.should.equal(100); + consumer.close(done); + }); + consumer.pipe( + through2.obj({ highWaterMark: 8 }, function (data, enc, cb) { + incrementPipeCount(data); + cb(null); + }) + ); + }); + }); + }); + it('should continue polling for new messages appended after consuming all existing messages', function (done) { + var client = new Client(); + var producer = new Producer(client); + producer.once('ready', function () { + createTopicAndProduceMessages(producer, APPEND_TOPIC_1, 20, function () { + var topics = [{ topic: APPEND_TOPIC_1 }]; + var options = { autoCommit: false, groupId: '_groupId_2_test' }; + var consumer = new ConsumerStream(client, topics, options); + var eventCounter = new EventCounter(); + var increment1 = eventCounter.createEventCounter('first', 20, function (error, firstEvents) { + should.not.exist(error); + firstEvents.events.length.should.equal(20); + firstEvents.events[0][0].value.should.equal('stream message 1'); + firstEvents.events[0][0].offset.should.equal(0); + firstEvents.events[19][0].value.should.equal('stream message 20'); + firstEvents.events[19][0].offset.should.equal(19); + var increment2 = eventCounter.createEventCounter('second', 20, function (error, secondEvents) { + should.not.exist(error); + secondEvents.count.should.equal(20); + secondEvents.events[0][0].value.should.equal('stream message 1'); + secondEvents.events[0][0].offset.should.equal(20); + secondEvents.events[19][0].value.should.equal('stream message 20'); + secondEvents.events[19][0].offset.should.equal(39); + consumer.close(done); + }); + consumer.on('data', increment2); + createTopicAndProduceMessages(producer, APPEND_TOPIC_1, 20); + }); + consumer.pipe( + through2.obj(function (data, enc, cb) { + increment1(data); + cb(null); + }) + ); + }); + }); + }); + describe('CommitStream', function () { + it('should instantiate a consumer stream and increment commit manually', function (done) { + const groupId = '_commitStream_1_test'; + const topic = COMMIT_STREAM_TOPIC_1; + var client = new Client(); + var producer = new Producer(client); + producer.once('ready', function () { + createTopicAndProduceMessages(producer, topic, 20, function () { + var options = { autoCommit: false, groupId }; + var consumer = new ConsumerStream(client, [topic], options); + var eventCounter = new EventCounter(); + let commitStream = consumer.createCommitStream({}); + var increment = eventCounter.createEventCounter('first', 20, function (error, events) { + if (error) { + throw error; + } + setImmediate(function () { + commitStream.commit(function () { + client.sendOffsetFetchRequest(groupId, commitStream.topicPartionOffsets, function (error, data) { + if (error) { + throw error; + } + data[topic][0].should.equal(20); + consumer.close(done); + }); + }); + }); + }); + consumer + .pipe( + through2.obj(function (data, enc, cb) { + increment(); + cb(null, data); + }) + ) + .pipe(commitStream); + }); + }); + }); + xit('should commit when the autocommit message count is reached', function (done) { + const groupId = '_commitStream_2_test'; + const topic = COMMIT_STREAM_TOPIC_2; + var client = new Client(); + var producer = new Producer(client); + producer.once('ready', function () { + createTopicAndProduceMessages(producer, topic, 20, function () { + var options = { autoCommit: true, autoCommitIntervalMs: false, autoCommitMsgCount: 18, groupId }; + var consumer = new ConsumerStream(client, [topic], options); + let commitStream = consumer.createCommitStream(); + commitStream.once('commitComplete', function (data) { + client.sendOffsetFetchRequest(groupId, commitStream.topicPartionOffsets, function (error, data) { + if (error) { + throw error; + } + data[topic][0].should.equal(18); + consumer.close(done); + }); + }); + consumer + .pipe( + through2.obj(function (data, enc, cb) { + cb(null, data); + }) + ) + .pipe(commitStream); + }); + }); + }); + xit('should autocommit after a given interval in milliseconds', function (done) { + const groupId = '_commitStream_3_test'; + const topic = COMMIT_STREAM_TOPIC_3; + var client = new Client(); + var producer = new Producer(client); + producer.once('ready', function () { + createTopicAndProduceMessages(producer, topic, 20, function () { + var options = { autoCommit: true, autoCommitIntervalMs: 5, groupId }; + var consumer = new ConsumerStream(client, [topic], options); + let commitStream = consumer.createCommitStream(); + commitStream.once('commitComplete', function (data) { + client.sendOffsetFetchRequest(groupId, commitStream.topicPartionOffsets, function (error, data) { + if (error) { + throw error; + } + data[topic][0].should.equal(20); + commitStream.clearInterval(); + consumer.close(done); + }); + }); + consumer + .pipe( + through2.obj(function (data, enc, cb) { + cb(null, data); + }) + ) + .pipe(commitStream); + }); + }); + }); + }); +}); diff --git a/node_modules/kafka-node/test/test.errors.js b/node_modules/kafka-node/test/test.errors.js new file mode 100644 index 0000000..9e40046 --- /dev/null +++ b/node_modules/kafka-node/test/test.errors.js @@ -0,0 +1,9 @@ +'use strict'; + +const errors = require('../lib/errors'); + +describe('Test Errors', function () { + it('should have right number of consumer group errors', function () { + errors.ConsumerGroupErrors.length.should.be.eql(7); + }); +}); diff --git a/node_modules/kafka-node/test/test.highlevelProducer.js b/node_modules/kafka-node/test/test.highlevelProducer.js new file mode 100644 index 0000000..397d87d --- /dev/null +++ b/node_modules/kafka-node/test/test.highlevelProducer.js @@ -0,0 +1,289 @@ +'use strict'; + +var kafka = require('..'); +var HighLevelProducer = kafka.HighLevelProducer; +var uuid = require('uuid'); +var KafkaClient = kafka.KafkaClient; +var KeyedMessage = kafka.KeyedMessage; +const _ = require('lodash'); +const assert = require('assert'); +const ConsumerGroup = kafka.ConsumerGroup; +const sendMessage = require('./helpers/sendMessage'); +var client, producer, noAckProducer, producerKeyed; + +describe('partitioner', function () { + let topic, consumerGroup, messages; + const createTopic = require('../docker/createTopic'); + + beforeEach(function () { + messages = _.times(3, uuid.v4); + topic = uuid.v4(); + + return createTopic(topic, 3, 1).then(function () { + return new Promise(function (resolve, reject) { + sendMessage(messages, topic, function (error) { + if (error) { + return reject(error); + } + resolve(); + }); + }); + }); + }); + + afterEach(function (done) { + consumerGroup && consumerGroup.close(done); + }); + + it('should distribute messages round-robin using cyclic', function (done) { + consumerGroup = new ConsumerGroup( + { + kafkaHost: '127.0.0.1:9092', + groupId: uuid.v4(), + fromOffset: 'earliest' + }, + topic + ); + const offsets = []; + consumerGroup.on('message', function (message) { + offsets.should.not.containEql(message.offset); + offsets.push(message.offset); + _.pull(messages, message.value); + if (messages.length === 0) { + done(); + } + }); + }); +}); + +[ + { + name: 'PLAINTEXT HighLevelProducer using KafkaClient', + useKafkaClient: true + }, + { + name: 'SSL Producer using KafkaClient', + useKafkaClient: true, + sslOptions: { + rejectUnauthorized: false + }, + suiteTimeout: 30000 + } +].forEach(function (testParameters) { + var TOPIC_POSTFIX = '_test_' + uuid.v4(); + var EXISTS_TOPIC_3 = '_exists_3' + TOPIC_POSTFIX; + + var sslOptions = testParameters.sslOptions; + var suiteTimeout = testParameters.suiteTimeout; + var suiteName = testParameters.name; + + describe(suiteName, function () { + before(function (done) { + if (suiteTimeout) { + this.timeout(suiteTimeout); + } + const kafkaHost = 'localhost:' + (sslOptions != null ? '9093' : '9092'); + client = new KafkaClient({ + kafkaHost: kafkaHost, + sslOptions: sslOptions + }); + + producer = new HighLevelProducer(client); + noAckProducer = new HighLevelProducer(client, { requireAcks: 0 }); + producerKeyed = new HighLevelProducer(client, { partitionerType: HighLevelProducer.PARTITIONER_TYPES.keyed }); + + producer.on('ready', function () { + producerKeyed.on('ready', function () { + producer.createTopics([EXISTS_TOPIC_3], true, function (err) { + if (err) return done(err); + done(); + }); + }); + }); + }); + + after(function (done) { + producer.close(done); + }); + + describe('#buildPayloads', function () { + function assertMessage (result, topic, partition, value) { + assert( + _.chain(result) + .find({ topic: topic, partition: partition }) + .result('messages') + .map('value') + .includes(value) + .value(), + `Value "${value}" is not in topic "${topic}" partition ${partition}` + ); + } + + it('should normalize payload by topic parition', function () { + const topicMetadata = { + jolly: [0, 1, 2], + christmas: [0, 1, 2], + coal: [0] + }; + + const payload = [ + { topic: 'jolly', partition: 0, messages: 'jolly-test-0' }, + { topic: 'jolly', partition: 1, messages: 'jolly-test-1' }, + { topic: 'jolly', partition: 2, messages: 'jolly-test-2' }, + { topic: 'jolly', partition: 1, messages: 'jolly-test-3' }, + { topic: 'christmas', partition: 0, messages: 'christmas-test-0' }, + { topic: 'christmas', partition: 1, messages: 'christmas-test-1' }, + { topic: 'christmas', partition: 2, messages: 'christmas-test-2' }, + { topic: 'christmas', partition: 0, messages: 'christmas-test-3' }, + { topic: 'christmas', partition: 1, messages: 'christmas-test-4' }, + { topic: 'christmas', partition: 2, messages: 'christmas-test-5' }, + { topic: 'coal', partition: 0, messages: 'coal-test' } + ]; + + const requestPayload = producer.buildPayloads(payload, topicMetadata); + + Object.keys(requestPayload).length.should.be.eql(7); + + assertMessage(requestPayload, 'jolly', 0, 'jolly-test-0'); + assertMessage(requestPayload, 'jolly', 1, 'jolly-test-1'); + assertMessage(requestPayload, 'jolly', 1, 'jolly-test-3'); + assertMessage(requestPayload, 'jolly', 2, 'jolly-test-2'); + + assertMessage(requestPayload, 'christmas', 0, 'christmas-test-0'); + assertMessage(requestPayload, 'christmas', 1, 'christmas-test-1'); + assertMessage(requestPayload, 'christmas', 2, 'christmas-test-2'); + assertMessage(requestPayload, 'christmas', 0, 'christmas-test-3'); + assertMessage(requestPayload, 'christmas', 1, 'christmas-test-4'); + assertMessage(requestPayload, 'christmas', 2, 'christmas-test-5'); + + assertMessage(requestPayload, 'coal', 0, 'coal-test'); + }); + }); + + describe('#send', function () { + before(function (done) { + // Ensure that first message gets the `0` + producer.send([{ topic: EXISTS_TOPIC_3, messages: '_initial' }], function (err, message) { + message.should.be.ok; + message[EXISTS_TOPIC_3].should.have.property('0', 0); + done(err); + }); + }); + + it('should send message successfully', function (done) { + producer.send([{ topic: EXISTS_TOPIC_3, messages: 'hello kafka' }], function (err, message) { + message.should.be.ok; + message[EXISTS_TOPIC_3]['0'].should.be.above(0); + done(err); + }); + }); + + it('should send buffer message successfully', function (done) { + var message = Buffer.from('hello kafka'); + producer.send([{ topic: EXISTS_TOPIC_3, messages: message }], function (err, message) { + message.should.be.ok; + message[EXISTS_TOPIC_3]['0'].should.be.above(0); + done(err); + }); + }); + + it('should send null message successfully', function (done) { + var message = null; + producer.send([{ topic: EXISTS_TOPIC_3, messages: message }], function (err, message) { + message.should.be.ok; + message[EXISTS_TOPIC_3]['0'].should.be.above(0); + done(err); + }); + }); + + it('should convert none buffer message to string', function (done) { + var message = -1; + producer.send([{ topic: EXISTS_TOPIC_3, messages: message }], function (err, message) { + message.should.be.ok; + message[EXISTS_TOPIC_3]['0'].should.be.above(0); + done(err); + }); + }); + + it('should send Message struct successfully', function (done) { + var message = new KeyedMessage('test-key', 'test-message'); + producer.send([{ topic: EXISTS_TOPIC_3, messages: message }], function (err, message) { + message.should.be.ok; + message[EXISTS_TOPIC_3]['0'].should.be.above(0); + done(err); + }); + }); + + it('should support multi messages in one topic', function (done) { + producer.send([{ topic: EXISTS_TOPIC_3, messages: ['hello kafka', 'hello kafka'] }], function (err, message) { + message.should.be.ok; + message[EXISTS_TOPIC_3]['0'].should.be.above(0); + done(err); + }); + }); + + it('should support snappy compression', function (done) { + producer.send( + [ + { + topic: EXISTS_TOPIC_3, + messages: ['hello kafka', 'hello kafka'], + attributes: 2 + } + ], + function (err, message) { + if (err) return done(err); + message.should.be.ok; + message[EXISTS_TOPIC_3]['0'].should.be.above(0); + done(); + } + ); + }); + + it('should send message without ack', function (done) { + noAckProducer.send( + [ + { + topic: EXISTS_TOPIC_3, + messages: 'hello kafka' + } + ], + function (err, message) { + if (err) return done(err); + message.result.should.equal('no ack'); + done(); + } + ); + }); + + it('should send message to specified partition even when producer configured with keyed partitioner', function (done) { + producerKeyed.send([{ key: '12345', partition: 0, topic: EXISTS_TOPIC_3, messages: 'hello kafka' }], function ( + err, + message + ) { + message.should.be.ok; + message[EXISTS_TOPIC_3]['0'].should.be.above(0); + done(err); + }); + }); + }); + + describe('#close', function () { + var client, producer; + + before(function (done) { + client = new KafkaClient(); + producer = new HighLevelProducer(client); + producer.on('ready', done); + }); + + after(function (done) { + producer.close(done); + }); + + it('should close successfully', function (done) { + producer.close(done); + }); + }); + }); +}); diff --git a/node_modules/kafka-node/test/test.kafka-node.js b/node_modules/kafka-node/test/test.kafka-node.js new file mode 100644 index 0000000..b611fb5 --- /dev/null +++ b/node_modules/kafka-node/test/test.kafka-node.js @@ -0,0 +1,14 @@ +'use strict'; + +var kafka = require('..'); +var Client = kafka.KafkaClient; + +const host = '127.0.0.1:9092'; + +var client = null; + +before(function (done) { + client = new Client({ kafkaHost: host }); + client.once('ready', done); + client.once('error', done); +}); diff --git a/node_modules/kafka-node/test/test.kafkaClient.js b/node_modules/kafka-node/test/test.kafkaClient.js new file mode 100644 index 0000000..4c56c41 --- /dev/null +++ b/node_modules/kafka-node/test/test.kafkaClient.js @@ -0,0 +1,1728 @@ +'use strict'; + +const proxyquire = require('proxyquire'); +const kafka = require('..'); +const Client = kafka.KafkaClient; +const sinon = require('sinon'); +const TimeoutError = require('../lib/errors/TimeoutError'); +const TopicsNotExistError = require('../lib/errors/TopicsNotExistError'); +const NotControllerError = require('../lib/errors/NotControllerError'); +const SaslAuthenticationError = require('../lib/errors/SaslAuthenticationError'); +const BrokerWrapper = require('../lib/wrapper/BrokerWrapper'); +const FakeSocket = require('./mocks/mockSocket'); +const should = require('should'); +const _ = require('lodash'); +const uuid = require('uuid'); +const BufferList = require('bl'); +const retry = require('retry'); + +describe('Kafka Client', function () { + describe('Discover Group Coordinator', function () { + let client; + beforeEach(function (done) { + client = new Client(); + client.once('connect', done); + }); + + afterEach(function (done) { + client.close(done); + }); + + it('#sendGroupCoordinatorRequest', function (done) { + var operation = retry.operation(); + operation.attempt(function () { + client.sendGroupCoordinatorRequest('ExampleTopic', function (error, response) { + if (operation.retry(error)) { + return; + } + should(error).be.null; + response.coordinatorPort.should.be.eql(9092); + response.coordinatorHost.should.be.eql('127.0.0.1'); + done(); + }); + }); + }); + }); + + describe('#handleReceivedData', function () { + let socket; + + beforeEach(function () { + socket = { + buffer: new BufferList() + }; + }); + + it('should always consume entire response even if handlers are missing', function () { + const fakeClient = { + invokeResponseCallback: sinon.stub().returns(null) + }; + + sinon.spy(socket.buffer, 'consume'); + sinon + .stub(socket.buffer, 'readUInt32BE') + .onFirstCall() + .returns(0); + sinon.stub(socket.buffer, 'shallowSlice').returns({ + readUInt32BE: sinon.stub().returns(25) + }); + + socket.buffer.append(Uint8Array.from([0, 0, 0, 0])); + Client.prototype.handleReceivedData.call(fakeClient, socket); + sinon.assert.calledOnce(socket.buffer.shallowSlice); + sinon.assert.calledOnce(socket.buffer.consume); + sinon.assert.calledOnce(fakeClient.invokeResponseCallback); + should(socket.waiting).be.empty; + }); + + it('should consume entire response if handlers are missing and set waiting to false for longpolling sockets', function () { + const fakeClient = { + invokeResponseCallback: sinon.stub().returns(null) + }; + + sinon.spy(socket.buffer, 'consume'); + sinon + .stub(socket.buffer, 'readUInt32BE') + .onFirstCall() + .returns(0); + sinon.stub(socket.buffer, 'shallowSlice').returns({ + readUInt32BE: sinon.stub().returns(25) + }); + + socket.longpolling = true; + socket.waiting = true; + + socket.buffer.append(Uint8Array.from([0, 0, 0, 0])); + Client.prototype.handleReceivedData.call(fakeClient, socket); + sinon.assert.calledOnce(socket.buffer.shallowSlice); + sinon.assert.calledOnce(socket.buffer.consume); + sinon.assert.calledOnce(fakeClient.invokeResponseCallback); + socket.waiting.should.be.false; + }); + + it('should early return when buffer is beyond offset', function () { + const fakeClient = { + invokeResponseCallback: function () {} + }; + + socket.buffer.append(Uint8Array.from([0, 0, 0])); + + const readSpy = sinon.spy(socket.buffer, 'readUInt32BE'); + Client.prototype.handleReceivedData.call(fakeClient, socket); + sinon.assert.notCalled(readSpy); + }); + + it('should early return when buffer is empty', function () { + const fakeClient = { + invokeResponseCallback: function () {} + }; + + const readSpy = sinon.spy(socket.buffer, 'readUInt32BE'); + Client.prototype.handleReceivedData.call(fakeClient, socket); + sinon.assert.notCalled(readSpy); + }); + }); + + describe('#parseHostList', function () { + it('initial hosts should be parsed if single host is provided', function () { + const client = new Client({ + autoConnect: false, + kafkaHost: 'localhost:9092' + }); + + client.initialHosts.should.not.be.empty; + client.initialHosts.length.should.be.eql(1); + client.initialHosts[0].host.should.be.eql('localhost'); + client.initialHosts[0].port.should.be.eql(9092); + }); + + it('initial hosts should be parsed if multiple hosts are provided', function () { + const client = new Client({ + autoConnect: false, + kafkaHost: 'localhost:9092,127.0.0.1:9093,192.168.1.0:9094' + }); + + client.initialHosts.should.not.be.empty; + client.initialHosts.should.be.eql([ + { + host: 'localhost', + port: 9092 + }, + { + host: '127.0.0.1', + port: 9093 + }, + { + host: '192.168.1.0', + port: 9094 + } + ]); + }); + }); + + describe('#createBroker', function () { + let sandbox, Client, mockSocket; + beforeEach(function () { + sandbox = sinon.sandbox.create(); + mockSocket = new FakeSocket(); + Client = proxyquire('../lib/kafkaClient', { + net: { + createConnection () { + return mockSocket; + } + } + }); + }); + + afterEach(function () { + sandbox.restore(); + }); + + it('should not emit connect event when fails to initializeBroker', function () { + const client = new Client({ autoConnect: false }); + sandbox.stub(client, 'initializeBroker').yields(new Error('fake error')); + sandbox.spy(client, 'emit'); + + client.createBroker('fakehost', 9092, true); + + mockSocket.emit('connect'); + sinon.assert.calledOnce(client.initializeBroker); + sinon.assert.notCalled(client.emit); + }); + + it('should not emit reconnect event when fails to initializeBroker', function () { + const client = new Client({ autoConnect: false }); + sandbox.stub(client, 'initializeBroker').yields(new Error('fake error')); + sandbox.spy(client, 'emit'); + + client.createBroker('fakehost', 9092, true); + + mockSocket.error = new Error('some socket error'); + + mockSocket.emit('connect'); + sinon.assert.calledOnce(client.initializeBroker); + sinon.assert.notCalled(client.emit); + }); + + it('should not reconnect when broker is no longer valid', function () { + sandbox.useFakeTimers(); + const client = new Client({ autoConnect: false }); + client.brokerMetadata = { + '1001': { + host: 'localhost', + port: 9092 + }, + '1002': { + host: 'kafkaServer', + port: 9092 + } + }; + sandbox.stub(client, 'reconnectBroker'); + client.createBroker('fakehost', 9092, true); + mockSocket.emit('end'); + sandbox.clock.tick(1000); + sinon.assert.notCalled(client.reconnectBroker); + }); + + it('should try reconnecting when client is initializing', function () { + sandbox.useFakeTimers(); + const client = new Client({ autoConnect: false }); + client.connecting = true; + client.brokerMetadata = {}; + sandbox.stub(client, 'reconnectBroker'); + client.createBroker('fakehost', 9092, true); + mockSocket.emit('end'); + sandbox.clock.tick(1000); + sinon.assert.calledOnce(client.reconnectBroker); + }); + + it('should schedule refresh of metadata when socket is closed', function (done) { + const client = new Client({ autoConnect: false }); + sandbox.stub(client, 'refreshBrokerMetadata').callsFake(done); + client.createBroker('fakehost', 9092, true); + mockSocket.emit('close', false); + }); + + it('should not schedule refresh of metadata when client is initalizing', function (done) { + const client = new Client({ autoConnect: false }); + client.connecting = true; + sandbox.stub(client, 'refreshBrokerMetadata'); + client.createBroker('fakehost', 9092, true); + mockSocket.emit('close', false); + setImmediate(function () { + sinon.assert.notCalled(client.refreshBrokerMetadata); + done(); + }); + }); + + it('should not schedule metadata refresh when broker is closed due to being idle', function () { + const client = new Client({ autoConnect: false }); + const brokerWrapper = client.createBroker('fakehost', 9092, true); + + sandbox.stub(brokerWrapper, 'isIdle').returns(true); + sandbox.stub(client, 'refreshBrokerMetadata'); + sandbox.useFakeTimers(); + + mockSocket.emit('close', false); + sandbox.clock.tick(); + sinon.assert.notCalled(client.refreshBrokerMetadata); + }); + + it('should reconnect when broker is closed due to being idle', function () { + const client = new Client({ autoConnect: false, reconnectOnIdle: true }); + const brokerWrapper = client.createBroker('fakehost', 9092, true); + + sandbox.stub(brokerWrapper, 'isIdle').returns(true); + sandbox.stub(client, 'isValidBroker').returns(true); + sandbox.stub(client, 'reconnectBroker'); + sandbox.useFakeTimers(); + + mockSocket.emit('close', false); + sandbox.clock.tick(1001); + sinon.assert.calledWithExactly(client.reconnectBroker, mockSocket); + }); + }); + + describe('#sendRequest', function () { + let sandbox; + before(function () { + sandbox = sinon.sandbox.create(); + }); + + afterEach(function () { + sandbox.restore(); + }); + + it('should call refreshBrokerMetadata if broker is not connected', function (done) { + const client = new Client({ + autoConnect: false, + kafkaHost: 'localhost:9092' + }); + + const fakeSocket = new FakeSocket(); + fakeSocket.destroyed = true; + + const fakeBroker = new BrokerWrapper(fakeSocket); + fakeBroker.apiSupport = {}; + + const BrokerNotAvailableError = require('../lib/errors').BrokerNotAvailableError; + + sandbox.stub(client, 'leaderByPartition').returns('1001'); + sandbox.stub(client, 'brokerForLeader').returns(fakeBroker); + sandbox.stub(client, 'refreshBrokerMetadata'); + + const request = { + type: 'produce', + data: { + payloads: [ + { + topic: 'test-topic', + partition: 0, + messages: [ + { + magic: 0, + attributes: 0, + key: 'test-key', + value: 'test-message', + timestamp: 1511365962702 + } + ] + } + ] + }, + args: [1, 100], + requireAcks: 1 + }; + + client.sendRequest(request, function (error) { + sinon.assert.calledOnce(client.refreshBrokerMetadata); + error.should.not.be.empty; + error.should.be.an.instanceOf(BrokerNotAvailableError); + done(); + }); + }); + }); + + describe('Versions', function () { + let client; + afterEach(function (done) { + client.close(done); + }); + + describe('#initializeBroker', function () { + it('should not call #getApiVersions if versions is disabled', function (done) { + client = new Client({ kafkaHost: '127.0.0.1:9092', autoConnect: false, versions: { disabled: true } }); + client.options.versions.disabled.should.be.true; + const brokerInitSpy = sinon.spy(client, 'initializeBroker'); + const apiVersionsSpy = sinon.spy(client, 'getApiVersions'); + client.connect(); + client.once('connect', function () { + sinon.assert.calledOnce(brokerInitSpy); + sinon.assert.notCalled(apiVersionsSpy); + brokerInitSpy.restore(); + apiVersionsSpy.restore(); + done(); + }); + }); + + it('should call #getApiVersions if versions is enabled', function (done) { + client = new Client({ kafkaHost: '127.0.0.1:9092', autoConnect: false }); + client.options.versions.disabled.should.be.false; + const brokerInitSpy = sinon.spy(client, 'initializeBroker'); + const apiVersionsSpy = sinon.spy(client, 'getApiVersions'); + client.connect(); + client.once('connect', function () { + sinon.assert.calledOnce(brokerInitSpy); + sinon.assert.calledOnce(apiVersionsSpy); + sinon.assert.callOrder(brokerInitSpy, apiVersionsSpy); + brokerInitSpy.restore(); + apiVersionsSpy.restore(); + done(); + }); + }); + + if (process.env.KAFKA_VERSION === '0.9') { + it('should return base support mapping', function (done) { + client = new Client({ kafkaHost: '127.0.0.1:9092' }); + client.once('connect', function () { + const broker = client.brokerForLeader(); + broker.isConnected().should.be.true; + broker.should.have.property('apiSupport'); + broker.apiSupport.should.be.type('object'); + _.forOwn(broker.apiSupport, function (support, api) { + if (support === null) { + return; + } + support.should.be.type('object'); + support.should.have.keys('min', 'max', 'usable'); + }); + done(); + }); + }); + } + }); + + describe('#getApiVersions', function () { + beforeEach(function (done) { + client = new Client({ kafkaHost: '127.0.0.1:9092' }); + client.once('connect', done); + }); + + if (process.env.KAFKA_VERSION === '0.9') { + it('#getApiVersions failure for 0.9', function (done) { + client.getApiVersions(client.brokerForLeader(), function (error, results) { + error.should.be.an.instanceOf(TimeoutError); + done(); + }); + }); + } else { + it('#getApiVersions returns results', function (done) { + client.getApiVersions(client.brokerForLeader(), function (error, results) { + should(results).not.be.empty; + _.forOwn(results, function (support, api) { + if (support === null) { + return; + } + support.should.have.keys('min', 'max', 'usable'); + }); + done(error); + }); + }); + } + }); + }); + + describe('#deleteDisconnected', function () { + let sandbox, client, fakeBroker; + + before(function () { + sandbox = sinon.sandbox.create(); + client = new Client({ kafkaHost: '127.0.0.1:9092', autoConnect: false, requestTimeout: 4000 }); + fakeBroker = new BrokerWrapper(new FakeSocket()); + fakeBroker.socket.addr = '127.0.0.1:9092'; + }); + + afterEach(function () { + sandbox.restore(); + }); + + it('should not delete if broker is still connected', function () { + sandbox.spy(client, 'getBrokers'); + + client.brokers[fakeBroker.socket.addr] = fakeBroker; + + sandbox.stub(fakeBroker, 'isConnected').returns(true); + client.deleteDisconnected(fakeBroker); + sinon.assert.notCalled(client.getBrokers); + sinon.assert.calledOnce(fakeBroker.isConnected); + + client.brokers.should.have.property(fakeBroker.socket.addr); + }); + + it('should throw if the broker is key does not match the instance', function () { + sandbox.spy(client, 'getBrokers'); + client.brokers[fakeBroker.socket.addr] = {}; + sandbox.stub(fakeBroker, 'isConnected').returns(false); + + should.throws(function () { + client.deleteDisconnected(fakeBroker); + }); + + sinon.assert.calledOnce(client.getBrokers); + sinon.assert.calledOnce(fakeBroker.isConnected); + + client.brokers.should.have.property(fakeBroker.socket.addr); + }); + + it('should delete broker if disconnected', function () { + sandbox.spy(client, 'getBrokers'); + + client.brokers[fakeBroker.socket.addr] = fakeBroker; + + sandbox.stub(fakeBroker, 'isConnected').returns(false); + client.deleteDisconnected(fakeBroker); + sinon.assert.calledOnce(client.getBrokers); + sinon.assert.calledOnce(fakeBroker.isConnected); + + client.brokers.should.not.have.property(fakeBroker.socket.addr); + }); + }); + + describe('#waitUntilReady', function () { + let sandbox, client, clock; + + before(function () { + sandbox = sinon.sandbox.create(); + clock = sandbox.useFakeTimers(); + client = new Client({ kafkaHost: '127.0.0.1:9092', autoConnect: false, requestTimeout: 4000 }); + }); + + afterEach(function () { + sandbox.restore(); + }); + + it('should yield error timeout if broker is not ready by requestTimeout', function (done) { + const fakeBroker = new BrokerWrapper(new FakeSocket()); + const readyKey = 'broker.host-ready'; + + sandbox.stub(fakeBroker, 'getReadyEventName').returns(readyKey); + sandbox.spy(client, 'removeListener'); + sandbox.spy(client, 'once'); + + client.waitUntilReady(fakeBroker, function (error) { + error.should.not.be.empty; + error.should.be.an.instanceOf(TimeoutError); + sinon.assert.calledOnce(fakeBroker.getReadyEventName); + sinon.assert.calledWith(client.removeListener, readyKey, sinon.match.func); + sinon.assert.calledWith(client.once, readyKey, sinon.match.func); + done(); + }); + + clock.tick(client.options.requestTimeout + 1); + }); + + it('should yield if broker is ready before requestTimeout', function (done) { + const fakeBroker = new BrokerWrapper(new FakeSocket()); + const readyKey = 'broker.host-ready'; + + sandbox.stub(fakeBroker, 'getReadyEventName').returns(readyKey); + sandbox.spy(client, 'removeListener'); + sandbox.spy(client, 'once'); + + client.waitUntilReady(fakeBroker, function (error) { + should(error).be.empty; + sinon.assert.calledOnce(fakeBroker.getReadyEventName); + sinon.assert.calledWith(client.once, readyKey, sinon.match.func); + done(); + }); + + clock.tick(client.options.requestTimeout - 1); + client.emit(readyKey); + }); + }); + + describe('#wrapTimeoutIfNeeded', function () { + let sandbox, wrapTimeoutIfNeeded, client, clock; + + beforeEach(function () { + sandbox = sinon.sandbox.create(); + clock = sandbox.useFakeTimers(); + client = { + unqueueCallback: sandbox.stub(), + options: { + requestTimeout: false + } + }; + wrapTimeoutIfNeeded = Client.prototype.wrapTimeoutIfNeeded.bind(client); + }); + + afterEach(function () { + sandbox.restore(); + }); + + it('should not wrap if there is not a this.options.requestTimeout', function () { + const myFn = function () {}; + const retFn = wrapTimeoutIfNeeded(1, 1, myFn); + myFn.should.be.exactly(retFn); + }); + + it('should not yield timeout if returned callback is called in time', function (done) { + client.options.requestTimeout = 400; + const retFn = wrapTimeoutIfNeeded(1, 1, done); + retFn.should.not.be.exactly(done); + clock.tick(300); + retFn(); + clock.tick(300); + }); + + it('should yield timeout error if not called by timeout', function (done) { + client.options.requestTimeout = 400; + function callback (error) { + error.should.be.an.instanceOf(TimeoutError); + error.message.should.be.exactly('Request timed out after 400ms'); + sinon.assert.calledWithExactly(client.unqueueCallback, 1, 10); + done(); + } + const retFn = wrapTimeoutIfNeeded(1, 10, callback); + retFn.should.not.be.exactly(callback); + clock.tick(400); + retFn(new Error('BAD')); + }); + }); + + describe('#setBrokerMetadata', function () { + let clock; + + beforeEach(function () { + clock = sinon.useFakeTimers(); + }); + + afterEach(function () { + clock.restore(); + }); + + it('should set new brokerMetadata field on client no emit', function () { + const client = new Client({ + autoConnect: false, + kafkaHost: 'Kafka-1.us-east-1.myapp.com:9093' + }); + + const brokerMetadata = { + '1': { nodeId: 1, host: 'Kafka-1.us-east-1.myapp.com', port: 9093 }, + '2': { nodeId: 2, host: 'Kafka-2.us-east-1.myapp.com', port: 9093 }, + '3': { nodeId: 3, host: 'Kafka-3.us-east-1.myapp.com', port: 9093 } + }; + + client.on('brokersChanged', function () { + throw new Error('should not emit'); + }); + + client.setBrokerMetadata(brokerMetadata); + client.brokerMetadata.should.be.eql(brokerMetadata); + client.brokerMetadataLastUpdate.should.be.eql(0); + clock.tick(100); + }); + + it('should set same brokerMetadata field on client no emit', function () { + const client = new Client({ + autoConnect: false, + kafkaHost: 'Kafka-1.us-east-1.myapp.com:9093' + }); + + const brokerMetadata = { + '1': { nodeId: 1, host: 'Kafka-1.us-east-1.myapp.com', port: 9093 }, + '2': { nodeId: 2, host: 'Kafka-2.us-east-1.myapp.com', port: 9093 }, + '3': { nodeId: 3, host: 'Kafka-3.us-east-1.myapp.com', port: 9093 } + }; + + client.brokerMetadata = brokerMetadata; + should(client.brokerMetadataLastUpdate).be.empty; + + client.on('brokersChanged', function () { + throw new Error('should not emit'); + }); + + client.setBrokerMetadata(brokerMetadata); + client.brokerMetadata.should.be.eql(brokerMetadata); + client.brokerMetadataLastUpdate.should.be.eql(0); + clock.tick(100); + }); + + it('should set different brokerMetadata field on client emit', function (done) { + const client = new Client({ + autoConnect: false, + kafkaHost: 'Kafka-1.us-east-1.myapp.com:9093' + }); + + const brokerMetadata = { + '1': { nodeId: 1, host: 'Kafka-1.us-east-1.myapp.com', port: 9093 }, + '2': { nodeId: 2, host: 'Kafka-2.us-east-1.myapp.com', port: 9093 }, + '3': { nodeId: 3, host: 'Kafka-3.us-east-1.myapp.com', port: 9093 } + }; + + client.brokerMetadata = _.clone(brokerMetadata); + should(client.brokerMetadataLastUpdate).be.empty; + + delete brokerMetadata['1']; + client.on('brokersChanged', done); + + client.setBrokerMetadata(brokerMetadata); + client.brokerMetadata.should.be.eql(brokerMetadata); + client.brokerMetadataLastUpdate.should.be.eql(0); + clock.tick(100); + }); + }); + + describe('#connect', function () { + let client; + + afterEach(function (done) { + client.close(done); + }); + + it('should connect plaintext', function (done) { + client = new Client({ + kafkaHost: 'localhost:9092' + }); + client.once('error', done); + client.once('ready', function () { + client.brokerMetadata.should.not.be.empty; + client.ready.should.be.true; + done(); + }); + }); + + it('should error when connecting to an invalid host', function (done) { + client = new Client({ + connectRetryOptions: { + retries: 0 + }, + kafkaHost: 'localhost:9095' + }); + + client.on('error', function (error) { + client.ready.should.be.false; + error.code.should.be.eql('ECONNREFUSED'); + done(); + }); + }); + + it('should connect SSL', function (done) { + client = new Client({ + kafkaHost: 'localhost:9093', + sslOptions: { + rejectUnauthorized: false + } + }); + client.once('error', done); + client.once('ready', function () { + client.ready.should.be.true; + client.brokerMetadata.should.not.be.empty; + done(); + }); + }); + + describe('using SASL authentication', function () { + before(function () { + // these tests should not run again Kafka 0.8 & 0.9 + const supportsSaslPlain = + !process.env.KAFKA_VERSION || (process.env.KAFKA_VERSION !== '0.8' && process.env.KAFKA_VERSION !== '0.9'); + if (!supportsSaslPlain) { + this.skip(); + } + }); + + it('should connect SASL/PLAIN', function (done) { + client = new Client({ + kafkaHost: 'localhost:9094', + sasl: { + mechanism: 'plain', + username: 'kafkanode', + password: 'kafkanode' + }, + connectRetryOptions: { + retries: 0 + } + }); + client.once('error', done); + client.once('ready', function () { + client.ready.should.be.true; + client.brokerMetadata.should.not.be.empty; + done(); + }); + }); + + it('should not connect SASL/PLAIN with bad credentials', function (done) { + client = new Client({ + kafkaHost: 'localhost:9094', + sasl: { + mechanism: 'plain', + username: 'kafkanode', + password: 'badpasswd' + }, + connectRetryOptions: { + retries: 0 + } + }); + client.once('error', function (err) { + if (err instanceof SaslAuthenticationError) { + // expected + done(); + } else { + done(err); + } + }); + client.once('ready', function () { + var err = new Error('expected error!'); + done(err); + }); + }); + }); + + describe('using SASL SCRAM authentication', function () { + before(function () { + // these tests should not run again Kafka 0.8 & 0.9 & 0.10 + const supportsSaslPlain = + !process.env.KAFKA_VERSION || (process.env.KAFKA_VERSION !== '0.8' && process.env.KAFKA_VERSION !== '0.9' && process.env.KAFKA_VERSION !== '0.10'); + if (!supportsSaslPlain) { + this.skip(); + } + }); + + it('should connect SASL/SCRAM-SHA-256', function (done) { + client = new Client({ + kafkaHost: 'localhost:9094', + sasl: { + mechanism: 'SCRAM-SHA-256', + username: 'kafkanode', + password: 'kafkanode' + }, + connectRetryOptions: { + retries: 0 + } + }); + client.once('error', done); + client.once('ready', function () { + client.ready.should.be.true; + client.brokerMetadata.should.not.be.empty; + done(); + }); + }); + + it('should connect SASL/SCRAM-SHA-512', function (done) { + client = new Client({ + kafkaHost: 'localhost:9094', + sasl: { + mechanism: 'scram-sha-512', + username: 'kafkanode', + password: 'kafkanode' + }, + connectRetryOptions: { + retries: 0 + } + }); + client.once('error', done); + client.once('ready', function () { + client.ready.should.be.true; + client.brokerMetadata.should.not.be.empty; + done(); + }); + }); + + it('should not connect SASL/SCRAM-SHA-256 with bad credentials', function (done) { + client = new Client({ + kafkaHost: 'localhost:9094', + sasl: { + mechanism: 'scram-sha-256', + username: 'kafkanode', + password: 'badpasswd' + }, + connectRetryOptions: { + retries: 0 + } + }); + client.once('error', function (err) { + if (err instanceof SaslAuthenticationError) { + // expected + done(); + } else { + done(err); + } + }); + client.once('ready', function () { + var err = new Error('expected error!'); + done(err); + }); + }); + + it('should not connect SASL/SCRAM-SHA-512 with bad credentials', function (done) { + client = new Client({ + kafkaHost: 'localhost:9094', + sasl: { + mechanism: 'scram-sha-256', + username: 'kafkanode', + password: 'badpasswd' + }, + connectRetryOptions: { + retries: 0 + } + }); + client.once('error', function (err) { + if (err instanceof SaslAuthenticationError) { + // expected + done(); + } else { + done(err); + } + }); + client.once('ready', function () { + var err = new Error('expected error!'); + done(err); + }); + }); + }); + }); + + describe('#updateMetadatas', function () { + let client, sandbox; + + const updatedPartialMetadata = [ + null, + { + metadata: { + topic1: { + '0': { + topic: 'topic1', + partition: 0, + leader: 1001 + }, + '1': { + topic: 'topic1', + partition: 1, + leader: 1001 + }, + '2': { + topic: 'topic1', + partition: 2, + leader: 1001 + } + } + } + } + ]; + + beforeEach(function () { + sandbox = sinon.sandbox.create(); + client = new Client({ + connectRetryOptions: { + retries: 0 + }, + autoConnect: false, + kafkaHost: 'localhost:9093', + sslOptions: { + rejectUnauthorized: false + } + }); + }); + + it('should extend topicMetadata by default instead of replace', function () { + client.topicMetadata = { + topic0: { + '0': { + topic: 'topic0', + partition: 0, + leader: 1001 + } + }, + topic1: { + '0': { + topic: 'topic1', + partition: 0, + leader: 1001 + } + } + }; + sandbox.stub(client, 'setBrokerMetadata'); + client.updateMetadatas(updatedPartialMetadata); + client.topicMetadata.topic0.should.not.be.empty; + client.topicMetadata.topic1.should.have.property('1', { + topic: 'topic1', + partition: 1, + leader: 1001 + }); + client.topicMetadata.topic1.should.have.property('2', { + topic: 'topic1', + partition: 2, + leader: 1001 + }); + sinon.assert.calledWithExactly(client.setBrokerMetadata, null); + }); + + it('should replace if second parameter replaceTopicMetadata is true', function () { + client.topicMetadata = { + topic0: { + '0': { + topic: 'topic0', + partition: 0, + leader: 1001 + } + }, + topic1: { + '0': { + topic: 'topic1', + partition: 0, + leader: 1001 + } + } + }; + sandbox.stub(client, 'setBrokerMetadata'); + + client.topicMetadata.should.not.be.empty; + + client.updateMetadatas(updatedPartialMetadata, true); + + client.topicMetadata.should.not.have.property('topic0'); + sinon.assert.calledWithExactly(client.setBrokerMetadata, null); + }); + }); + + describe('#refreshBrokerMetadata', function () { + let sandbox, client; + + beforeEach(function () { + sandbox = sinon.sandbox.create(); + client = new Client({ + connectRetryOptions: { + retries: 0 + }, + autoConnect: false, + kafkaHost: 'localhost:9093', + sslOptions: { + rejectUnauthorized: false + } + }); + }); + + afterEach(function () { + sandbox.restore(); + }); + + it('should refresh broker metadata using available broker', function (done) { + const fakeBroker = new BrokerWrapper(new FakeSocket()); + const fakeDeadBroker = new BrokerWrapper(new FakeSocket()); + + const metadata = [ + { + '1': { nodeId: 1, host: 'Kafka-1.us-east-1.myapp.com', port: 9093 }, + '2': { nodeId: 2, host: 'Kafka-2.us-east-1.myapp.com', port: 9093 }, + '3': { nodeId: 3, host: 'Kafka-3.us-east-1.myapp.com', port: 9093 } + }, + { metadata: {} } + ]; + + client.brokerMetadata = _.clone(metadata[0]); + + const newBrokerKey = `${metadata[0][2].host}:${metadata[0][2].port}`; + const deadBrokerKey = `${metadata[0][1].host}:${metadata[0][1].port}`; + + client.brokers[deadBrokerKey] = fakeDeadBroker; + client.brokers[newBrokerKey] = fakeBroker; + + delete metadata[0]['1']; + + Object.assign(fakeBroker.socket, metadata['2']); + + sandbox.stub(client, 'getAvailableBroker').yields(null, fakeBroker); + sandbox.stub(client, 'loadMetadataFrom').yields(null, metadata); + sandbox.spy(client, 'updateMetadatas'); + sandbox.spy(client, 'refreshBrokers'); + + should(client.refreshingMetadata).be.empty; + + client.refreshBrokerMetadata(function (error) { + sinon.assert.calledOnce(client.getAvailableBroker); + sinon.assert.calledWith(client.loadMetadataFrom, fakeBroker, sinon.match.func); + sinon.assert.calledWith(client.updateMetadatas, metadata); + sinon.assert.calledOnce(client.refreshBrokers); + sinon.assert.callOrder( + client.getAvailableBroker, + client.loadMetadataFrom, + client.updateMetadatas, + client.refreshBrokers + ); + + client.brokers.should.have.property(newBrokerKey).and.be.exactly(fakeBroker); + client.brokers.should.not.have.property(deadBrokerKey); + + done(error); + }); + }); + + it('should emit an error', function (done) { + const expectedError = new Error('Unable to find available brokers to try'); + sandbox.stub(client, 'getAvailableBroker').yields(expectedError); + sandbox.stub(client, 'loadMetadataFrom'); + sandbox.stub(client, 'updateMetadatas'); + sandbox.stub(client, 'refreshBrokers'); + + client.on('error', function (error) { + error.should.be.an.instanceOf(Error); + error.nested.should.be.eql(expectedError); + done(); + }); + + client.refreshBrokerMetadata(); + }); + + it('should not perform refreshBrokerMetadata if one is in progress', function () { + sandbox.stub(client, 'getAvailableBroker'); + sandbox.stub(client, 'loadMetadataFrom'); + sandbox.stub(client, 'updateMetadatas'); + sandbox.stub(client, 'refreshBrokers'); + client.refreshingMetadata = true; + + client.refreshBrokerMetadata(); + + client.refreshingMetadata.should.be.true; + sinon.assert.notCalled(client.getAvailableBroker); + sinon.assert.notCalled(client.loadMetadataFrom); + sinon.assert.notCalled(client.updateMetadatas); + sinon.assert.notCalled(client.refreshBrokers); + }); + + it('should not perform refreshBrokerMetadata if consumer is closing', function () { + sandbox.stub(client, 'getAvailableBroker'); + sandbox.stub(client, 'loadMetadataFrom'); + sandbox.stub(client, 'updateMetadatas'); + sandbox.stub(client, 'refreshBrokers'); + client.closing = true; + + client.refreshBrokerMetadata(); + + client.closing.should.be.true; + sinon.assert.notCalled(client.getAvailableBroker); + sinon.assert.notCalled(client.loadMetadataFrom); + sinon.assert.notCalled(client.updateMetadatas); + sinon.assert.notCalled(client.refreshBrokers); + }); + }); + + describe('Verify Timeout', function () { + let sandbox; + + beforeEach(function () { + sandbox = sinon.sandbox.create(); + }); + + afterEach(function () { + sandbox.restore(); + }); + + it('should timeout when connect is not emitted', function (done) { + const clock = sandbox.useFakeTimers(); + const client = new Client({ + connectRetryOptions: { + retries: 0 + }, + autoConnect: false, + kafkaHost: 'localhost:9093', + sslOptions: { + rejectUnauthorized: false + } + }); + + const fakeSocket = new FakeSocket(); + + sandbox.spy(fakeSocket, 'destroy'); + sandbox.spy(fakeSocket, 'end'); + sandbox.spy(fakeSocket, 'unref'); + + sandbox.stub(client, 'setupBroker').returns({ + socket: fakeSocket + }); + + client.connect(); + client.once('error', function (error) { + error.should.be.an.instanceOf(TimeoutError); + fakeSocket.closing.should.be.true; + sinon.assert.callOrder(fakeSocket.end, fakeSocket.destroy, fakeSocket.unref); + done(); + }); + + clock.tick(10000); + }); + }); + + describe('#topicExists', function () { + const createTopic = require('../docker/createTopic'); + let sandbox, client; + + beforeEach(function (done) { + sandbox = sinon.sandbox.create(); + client = new Client({ + kafkaHost: 'localhost:9092' + }); + client.once('ready', done); + }); + + afterEach(function (done) { + sandbox.restore(); + client.close(done); + }); + + it('should not yield error when single topic exists', function (done) { + const topic = uuid.v4(); + + createTopic(topic, 1, 1).then(function () { + client.topicExists([topic], done); + }); + }); + + it('should yield error when given group of topics do not exist', function (done) { + sandbox.spy(client, 'loadMetadataForTopics'); + sandbox.spy(client, 'updateMetadatas'); + + const nonExistantTopics = _.times(3, () => uuid.v4()); + + client.topicExists(nonExistantTopics, function (error) { + error.should.be.an.instanceOf(TopicsNotExistError); + sinon.assert.calledOnce(client.updateMetadatas); + sinon.assert.calledWith(client.loadMetadataForTopics, []); + sinon.assert.callOrder(client.loadMetadataForTopics, client.updateMetadatas); + error.topics.should.be.eql(nonExistantTopics); + done(); + }); + }); + }); + + describe('#getListGroups', function () { + let client; + + afterEach(function (done) { + client.close(done); + }); + + it('should error if the client is not ready', function (done) { + client = new Client({ + kafkaHost: 'localhost:9092', + autoConnect: false + }); + client.getListGroups(function (error, res) { + error.should.be.an.instanceOf(Error); + error.message.should.be.exactly('Client is not ready (getListGroups)'); + done(); + }); + }); + }); + + describe('#createTopics', function () { + let client; + + before(function () { + if (process.env.KAFKA_VERSION === '0.9' || process.env.KAFKA_VERSION === '0.10') { + return this.skip(); + } + }); + + beforeEach(function (done) { + client = new Client({ + kafkaHost: 'localhost:9092' + }); + client.once('ready', done); + }); + + afterEach(function (done) { + client.close(done); + }); + + it('should create given topics', function (done) { + const topic1 = uuid.v4(); + const topic1ReplicationFactor = 1; + const topic1Partitions = 5; + const topic2 = uuid.v4(); + const topic2ReplicationFactor = 1; + const topic2Partitions = 1; + + client.createTopics( + [ + { + topic: topic1, + partitions: topic1Partitions, + replicationFactor: topic1ReplicationFactor + }, + { + topic: topic2, + partitions: topic2Partitions, + replicationFactor: topic2ReplicationFactor + } + ], + (error, result) => { + should.not.exist(error); + result.should.be.empty; + + // Verify topics were properly created with partitions + replication factor by fetching metadata again + const verifyPartitions = (topicMetadata, expectedPartitionCount, expectedReplicatonfactor) => { + for (let i = 0; i < expectedPartitionCount; i++) { + topicMetadata[i].partition.should.be.exactly(i); + topicMetadata[i].replicas.length.should.be.exactly(expectedReplicatonfactor); + } + }; + + client.loadMetadataForTopics([topic1, topic2], (error, result) => { + should.not.exist(error); + verifyPartitions(result[1].metadata[topic1], topic1Partitions, topic1ReplicationFactor); + verifyPartitions(result[1].metadata[topic2], topic2Partitions, topic2ReplicationFactor); + done(); + }); + } + ); + }); + + it('should return topic creation errors', function (done) { + const topic = uuid.v4(); + // Only 1 broker is available under test, so a replication factor > 1 is not possible + const topicReplicationFactor = 2; + const topicPartitions = 5; + + client.createTopics( + [ + { + topic: topic, + partitions: topicPartitions, + replicationFactor: topicReplicationFactor + } + ], + (error, result) => { + should.not.exist(error); + result.should.have.length(1); + result[0].topic.should.be.exactly(topic); + result[0].error.toLowerCase().should.startWith('replication factor: 2 larger than available brokers: 1'); + done(); + } + ); + }); + + it('should create topics with config entries', function (done) { + const topic1 = uuid.v4(); + const topic1ReplicationFactor = 1; + const topic1Partitions = 5; + + client.createTopics( + [ + { + topic: topic1, + partitions: topic1Partitions, + replicationFactor: topic1ReplicationFactor, + configEntries: [ + { + name: 'compression.type', + value: 'gzip' + }, + { + name: 'min.compaction.lag.ms', + value: '50' + } + ] + } + ], + (error, result) => { + should.not.exist(error); + result.should.be.empty; + + const resource = { + resourceType: 'topic', + resourceName: topic1, + configNames: [] + }; + + const payload = { + resources: [resource] + }; + + client.describeConfigs(payload, (error, result) => { + should.not.exist(error); + result[0].resourceName.should.be.exactly(topic1); + result[0].configEntries + .filter(c => { + return c.configName === 'compression.type'; + })[0] + .configValue.should.be.exactly('gzip'); + result[0].configEntries + .filter(c => { + return c.configName === 'min.compaction.lag.ms'; + })[0] + .configValue.should.be.exactly('50'); + done(); + }); + } + ); + }); + + it('should return topic creation errors with invalid config entries', function (done) { + const topic1 = uuid.v4(); + const topic1ReplicationFactor = 1; + const topic1Partitions = 5; + + client.createTopics( + [ + { + topic: topic1, + partitions: topic1Partitions, + replicationFactor: topic1ReplicationFactor, + configEntries: [ + { + name: 'compression.ty', + value: 'gzip' + } + ] + } + ], + (error, result) => { + should.not.exist(error); + result.should.have.length(1); + result[0].topic.should.be.exactly(topic1); + result[0].error.toLowerCase().should.startWith('unknown topic config name: compression.ty'); + done(); + } + ); + }); + + it('should create topics with explicit replica assignments if both simple and explicit assignment is provided', function (done) { + const topic1 = uuid.v4(); + const topic1ReplicationFactor = 1; + const topic1Partitions = 5; + + client.createTopics( + [ + { + topic: topic1, + partitions: topic1Partitions, + replicationFactor: topic1ReplicationFactor, + replicaAssignment: [ + { + partition: 0, + replicas: [1001] + }, + { + partition: 1, + replicas: [1001] + } + ] + } + ], + (error, result) => { + should.not.exist(error); + result.should.be.empty; + + client.loadMetadataForTopics([topic1], (error, result) => { + should.not.exist(error); + + const topicMetadata = result[1].metadata[topic1]; + Object.keys(topicMetadata).should.have.length(2); + topicMetadata['0'].partition.should.be.exactly(0); + topicMetadata['0'].replicas.should.have.length(1); + topicMetadata['1'].partition.should.be.exactly(1); + topicMetadata['1'].replicas.should.have.length(1); + done(); + }); + } + ); + }); + + it('should create topics with replica assignments if only explicit assignment is provided', function (done) { + const topic1 = uuid.v4(); + + client.createTopics( + [ + { + topic: topic1, + replicaAssignment: [ + { + partition: 0, + replicas: [1001] + }, + { + partition: 1, + replicas: [1001] + } + ] + } + ], + (error, result) => { + should.not.exist(error); + result.should.be.empty; + + client.loadMetadataForTopics([topic1], (error, result) => { + should.not.exist(error); + + const topicMetadata = result[1].metadata[topic1]; + Object.keys(topicMetadata).should.have.length(2); + topicMetadata['0'].partition.should.be.exactly(0); + topicMetadata['0'].replicas.should.have.length(1); + topicMetadata['1'].partition.should.be.exactly(1); + topicMetadata['1'].replicas.should.have.length(1); + done(); + }); + } + ); + }); + }); + + describe('#wrapControllerCheckIfNeeded', function () { + let client, sandbox; + + beforeEach(function (done) { + if (process.env.KAFKA_VERSION === '0.9') { + return this.skip(); + } + + sandbox = sinon.sandbox.create(); + client = new Client({ + kafkaHost: 'localhost:9092' + }); + client.once('ready', done); + }); + + afterEach(function (done) { + sandbox.restore(); + client.close(done); + }); + + it('should not wrap again if already wrapped', function () { + const fn = _.noop; + + const wrapped = client.wrapControllerCheckIfNeeded('', [], fn); + const secondWrapped = client.wrapControllerCheckIfNeeded('', [], wrapped); + + wrapped.should.be.exactly(secondWrapped); + }); + + it('should wrap if not already wrapped', function () { + const fn = _.noop; + + const wrapped = client.wrapControllerCheckIfNeeded(_.noop, _.noop, [], fn); + + wrapped.should.not.be.exactly(fn); + }); + + it('should set controller id to null if NotControllerError was returned once', function () { + const fn = _.noop; + const requestType = 'createTopics'; + const wrapped = client.wrapControllerCheckIfNeeded(requestType, [], fn); + const setControllerIdSpy = sandbox.spy(client, 'setControllerId'); + sandbox.stub(client, 'sendControllerRequest'); + + wrapped(new NotControllerError('not controller')); + + sinon.assert.calledOnce(setControllerIdSpy); + sinon.assert.alwaysCalledWithExactly(setControllerIdSpy, null); + }); + + it('should send controller request again if NotControllerError was returned once', function () { + var args = []; + const requestType = 'createTopics'; + const fn = _.noop; + const wrapped = client.wrapControllerCheckIfNeeded(requestType, args, fn); + const setControllerIdSpy = sandbox.spy(client, 'setControllerId'); + const sendControllerRequestSpy = sandbox.stub(client, 'sendControllerRequest'); + + wrapped(new NotControllerError('not controller')); + + sinon.assert.calledOnce(setControllerIdSpy); + sinon.assert.alwaysCalledWithExactly(setControllerIdSpy, null); + sinon.assert.calledOnce(sendControllerRequestSpy); + sinon.assert.alwaysCalledWithExactly(sendControllerRequestSpy, requestType, args, wrapped); + }); + + it('should set controller id to null and call original callback if NotControllerError was returned on second try', function () { + const fnSpy = sandbox.spy(); + const requestType = 'createTopics'; + const wrapped = client.wrapControllerCheckIfNeeded(requestType, [], fnSpy); + const setControllerIdSpy = sandbox.spy(client, 'setControllerId'); + sandbox.stub(client, 'sendControllerRequest'); + + wrapped(new NotControllerError('not controller')); + wrapped(new NotControllerError('not controller')); + + sinon.assert.calledTwice(setControllerIdSpy); + sinon.assert.alwaysCalledWithExactly(setControllerIdSpy, null); + sinon.assert.calledOnce(fnSpy); + }); + + it('should call original callback if another error was returned', function () { + const fnSpy = sandbox.spy(); + const requestType = 'createTopics'; + const wrapped = client.wrapControllerCheckIfNeeded(requestType, [], fnSpy); + const setControllerIdSpy = sandbox.spy(client, 'setControllerId'); + + wrapped(new TimeoutError('operation timed out')); + + sinon.assert.notCalled(setControllerIdSpy); + sinon.assert.calledOnce(fnSpy); + }); + + it('should call original callback if no error was returned', function () { + const fnSpy = sandbox.spy(); + const requestType = 'createTopics'; + const wrapped = client.wrapControllerCheckIfNeeded(requestType, [], fnSpy); + const setControllerIdSpy = sandbox.spy(client, 'setControllerId'); + const expectedResult = []; + + wrapped(null, expectedResult); + + sinon.assert.notCalled(setControllerIdSpy); + sinon.assert.calledOnce(fnSpy); + sinon.assert.alwaysCalledWith(fnSpy, null, expectedResult); + }); + }); + + describe('#loadMetadataForTopics', function () { + it('should request metadata from correct broker after ready', function (done) { + const client = new Client({ autoConnect: false }); + const brokerAddr = uuid.v4(); + + const brokerForLeaderStub = sinon.stub(client, 'brokerForLeader'); + sinon.spy(client, 'waitUntilReady'); + + const firstBroker = new BrokerWrapper(new FakeSocket()); + const secondBroker = new BrokerWrapper(new FakeSocket()); + + firstBroker.socket.addr = brokerAddr; + secondBroker.socket.addr = brokerAddr; + + brokerForLeaderStub.onFirstCall().returns(firstBroker); + brokerForLeaderStub.onSecondCall().returns(secondBroker); + + client.connecting = true; + client.loadMetadataForTopics([], function (error, result) { + if (error) { + return done(error); + } + sinon.assert.calledTwice(brokerForLeaderStub); + done(null); + }); + + firstBroker.socket.destroyed = true; + secondBroker.apiSupport = { + metadata: { + usable: 0 + } + }; + + sinon.stub(client, 'queueCallback').callsFake(function (socket, correlationId, coderAndCb) { + setImmediate(function () { + coderAndCb[1](null); + }); + }); + + client.emit(firstBroker.getReadyEventName()); + }); + }); + + describe('#getController', function () { + it('should use cached controller details even if controllerId is 0', function (done) { + const client = new Client({ autoConnect: false }); + + client.brokerMetadata = { + '0': { + host: 'fake-host', + port: 1234 + } + }; + + client.clusterMetadata = { + controllerId: 0 + }; + + const broker = uuid.v4(); + + sinon + .stub(client, 'getBroker') + .withArgs('fake-host', 1234) + .returns(broker); + + const loadMetadataStub = sinon.stub(client, 'loadMetadata').yieldsAsync(null, [{}, {}]); + + client.getController(function (error, broker, controllerId) { + sinon.assert.notCalled(loadMetadataStub); + sinon.assert.calledOnce(client.getBroker); + broker.should.be.eql(broker); + done(error); + }); + }); + }); + + describe('#sendControllerRequest', function () { + let client, sandbox; + + beforeEach(function (done) { + if (process.env.KAFKA_VERSION === '0.9') { + return this.skip(); + } + + sandbox = sinon.sandbox.create(); + client = new Client({ + kafkaHost: 'localhost:9092' + }); + client.once('ready', done); + }); + + afterEach(function (done) { + sandbox.restore(); + client.close(done); + }); + + it('should wrap callback', function () { + const requestType = 'createTopics'; + const fakeBroker = new BrokerWrapper(new FakeSocket()); + sandbox.stub(client, 'getController').yields(null, fakeBroker, 1); + sandbox.stub(client, 'queueCallback'); + const wrapControllerSpy = sandbox.spy(client, 'wrapControllerCheckIfNeeded'); + const callbackSpy = sandbox.spy(); + + client.sendControllerRequest(requestType, [], callbackSpy); + + sinon.assert.calledOnce(wrapControllerSpy); + }); + + it('should be called twice when NotController error was returned', function () { + const requestType = 'createTopics'; + const fakeBroker = new BrokerWrapper(new FakeSocket()); + sandbox.stub(client, 'getController').yields(null, fakeBroker, 1); + sandbox.stub(client, 'sendRequestToBroker').callsFake((brokerId, requestType, args, callback) => { + callback(new NotControllerError('not controller')); + }); + const callbackSpy = sandbox.spy(); + const sendControllerRequestSpy = sandbox.spy(client, 'sendControllerRequest'); + + client.sendControllerRequest(requestType, [], callbackSpy); + + sinon.assert.calledTwice(sendControllerRequestSpy); + }); + + it('should send request to controller', function () { + const requestType = 'createTopics'; + const fakeBroker = new BrokerWrapper(new FakeSocket()); + sandbox.stub(client, 'getController').yields(null, fakeBroker, 1); + const sendRequestSpy = sandbox.stub(client, 'sendRequestToBroker'); + const args = []; + const callback = _.noop; + + client.sendControllerRequest(requestType, args, callback); + + sinon.assert.calledOnce(sendRequestSpy); + }); + + it('should return error if controller request fails', function () { + const requestType = 'createTopics'; + const error = new TimeoutError('operation timed out'); + sandbox.stub(client, 'getController').yields(error); + const callbackSpy = sandbox.spy(); + + client.sendControllerRequest(requestType, null, callbackSpy); + + sinon.assert.calledOnce(callbackSpy); + sinon.assert.alwaysCalledWithExactly(callbackSpy, error); + }); + }); +}); diff --git a/node_modules/kafka-node/test/test.logging.js b/node_modules/kafka-node/test/test.logging.js new file mode 100644 index 0000000..fb7e273 --- /dev/null +++ b/node_modules/kafka-node/test/test.logging.js @@ -0,0 +1,23 @@ +'use strict'; + +const createLogger = require('../lib/logging'); +const logging = require('../logging'); +const sinon = require('sinon'); + +describe('logging', function () { + it('should expose a setLoggerProvider function', function () { + logging.setLoggerProvider.should.be.instanceOf(Function); + }); + + it('should create logger via custom logger provider', function () { + const provider = sinon.stub(); + const loggerName = 'kafka-node:consumer'; + const loggerImpl = {}; + provider.withArgs(loggerName).returns(loggerImpl); + logging.setLoggerProvider(provider); + + const logger = createLogger(loggerName); + + logger.should.equal(loggerImpl); + }); +}); diff --git a/node_modules/kafka-node/test/test.offset.js b/node_modules/kafka-node/test/test.offset.js new file mode 100644 index 0000000..98aaec4 --- /dev/null +++ b/node_modules/kafka-node/test/test.offset.js @@ -0,0 +1,215 @@ +'use strict'; + +var libPath = process.env['kafka-cov'] ? '../lib-cov/' : '../lib/'; +var Producer = require(libPath + 'producer'); +var Offset = require(libPath + 'offset'); +var Client = require(libPath + 'kafkaClient'); +var ConsumerGroup = require(libPath + 'consumerGroup'); +const uuid = require('uuid'); + +var client, producer, offset; + +describe('Offset', function () { + before(function (done) { + client = new Client(); + producer = new Producer(client); + producer.on('ready', function () { + producer.createTopics(['_exist_topic_3_test'], true, function (err) { + done(err); + }); + }); + + offset = new Offset(client); + }); + + after(function (done) { + producer.close(done); + }); + + describe('#fetch', function () { + it('should return offset of the topics', function (done) { + var topic = '_exist_topic_3_test'; + var topics = [{ topic: topic }]; + offset.fetch(topics, function (err, data) { + var offsets = data[topic][0]; + offsets.should.be.an.instanceOf(Array); + offsets.length.should.equal(1); + done(err); + }); + }); + + it('should return earliest offset of the topics', function (done) { + var topic = '_exist_topic_3_test'; + var topics = [{ topic: topic, time: -2 }]; + offset.fetch(topics, function (err, data) { + var offsets = data[topic][0]; + offsets.should.be.an.instanceOf(Array); + offsets.length.should.above(0); + done(err); + }); + }); + + it('should return latest offset of the topics', function (done) { + var topic = '_exist_topic_3_test'; + var topics = [{ topic: topic, time: -1 }]; + offset.fetch(topics, function (err, data) { + var offsets = data[topic][0]; + offsets.should.be.an.instanceOf(Array); + offsets.length.should.above(0); + done(err); + }); + }); + + it('should keeping calling fetch until offset is ready', function (done) { + var topic = '_exist_topic_3_test'; + var topics = [{ topic: topic }]; + offset.fetch(topics, done); + }); + }); + + describe('#commit', function () { + it('should commit successfully', function (done) { + var topic = '_exist_topic_3_test'; + var topics = [{ topic: topic, offset: 10 }]; + offset.commit('_groupId_commit_test', topics, function (err, data) { + data.should.be.ok; + Object.keys(data)[0].should.equal(topic); + done(err); + }); + }); + + it('should keep calling commit until offset is ready', function (done) { + var topic = '_exist_topic_3_test'; + var topics = [{ topic: topic, offset: 10 }]; + offset.commit('_groupId_commit_test', topics, done); + }); + }); + + describe('#fetchCommits', function () { + it('should get last committed offset of the consumer group', function (done) { + var topic = '_exist_topic_3_test'; + var topics = [{ topic: topic, offset: 10 }]; + offset.fetchCommits('_groupId_commit_1_test', topics, function (err, data) { + data.should.be.ok; + Object.keys(data)[0].should.equal(topic); + data[topic][0].should.equal(-1); + done(err); + }); + }); + + it('should keep calling fetchCommits until offset is ready', function (done) { + var topic = '_exist_topic_3_test'; + var topics = [{ topic: topic, offset: 10 }]; + offset.fetchCommits('_groupId_commit_1_test', topics, done); + }); + }); + + describe('#fetchCommitsV1', function () { + var topic, topics, groupId, expectedCommittedOffset; + topic = `_exist_topic_3_test`; + topics = [{ topic: topic, partition: 0 }]; + groupId = `_groupId_commit_v1_test`; + before(function (done) { + producer.send([{ topic, messages: ['firstMessage'] }], error => { + if (error) done(error); + createCGandCommitToLatestOffset(groupId, topic, (err, highWaterOffset) => { + expectedCommittedOffset = highWaterOffset; + done(err); + }); + }); + }); + + it('should return -1 when the consumer group has no commits on the broker', function (done) { + var groupIdNoCommits = groupId + '2'; + offset.fetchCommitsV1(groupIdNoCommits, topics, function (err, data) { + data.should.be.ok; + Object.keys(data)[0].should.equal(topic); + data[topic][0].should.equal(-1); + done(err); + }); + }); + + it('should get the last committed offset consumer group on the broker', function (done) { + offset.fetchCommitsV1(groupId, topics, function (err, data) { + data.should.be.ok; + Object.keys(data)[0].should.equal(topic); + data[topic][0].should.equal(expectedCommittedOffset); + done(err); + }); + }); + + it('should keep calling fetchCommits until offset is ready', function (done) { + var topic = '_exist_topic_3_test'; + var topics = [{ topic: topic, offset: 10 }]; + offset.fetchCommitsV1('_groupId_commit_1_test', topics, done); + }); + }); + + describe('#fetchEarliestOffsets', function () { + it('should callback with error if topic does not exist', function (done) { + offset.fetchEarliestOffsets([uuid.v4()], function (error) { + error.should.be.an.instanceOf(Error); + error.message.should.be.exactly('Topic(s) does not exist'); + done(); + }); + }); + }); + + describe('#fetchLatestOffsets', function () { + it('should callback with error if topic does not exist', function (done) { + offset.fetchLatestOffsets([uuid.v4()], function (error) { + error.should.be.an.instanceOf(Error); + error.message.should.be.exactly('Topic(s) does not exist'); + done(); + }); + }); + + it('should get latest kafka offsets for all topics passed in', function (done) { + var topic = '_exist_topic_3_test'; + var topics = [topic]; + var partition = 0; + offset.fetch([{ topic: topic, time: -1 }], function (err, results) { + if (err) return done(err); + var latestOffset = results[topic][partition][0]; + offset.fetchLatestOffsets(topics, function (err, offsets) { + if (err) return done(err); + offsets[topic][partition].should.equal(latestOffset); + done(); + }); + }); + }); + + it('should keep calling fetchLatestOffsets until offset is ready', function (done) { + var topic = '_exist_topic_3_test'; + var topics = [topic]; + offset.fetchLatestOffsets(topics, done); + }); + }); +}); + +const createCGandCommitToLatestOffset = (groupId, topic, cb) => { + try { + var consumerGroupOptions = { + groupId: groupId, + fromOffset: 'earliest', + kafkaHost: 'localhost:9092', + autoCommitIntervalMs: 1, + autoCommit: true + }; + var consumerGroup = new ConsumerGroup(consumerGroupOptions, topic); + consumerGroup.on('message', message => { + if (message.offset === message.highWaterOffset - 1) { + setTimeout(() => { + consumerGroup.close(true, () => { + return cb(null, message.highWaterOffset); + }); + }, 0); + } + }); + consumerGroup.on('error', err => { + return cb(err); + }); + } catch (e) { + return cb(e); + } +}; diff --git a/node_modules/kafka-node/test/test.partitioner.js b/node_modules/kafka-node/test/test.partitioner.js new file mode 100644 index 0000000..170de53 --- /dev/null +++ b/node_modules/kafka-node/test/test.partitioner.js @@ -0,0 +1,107 @@ +'use strict'; + +var _ = require('lodash'); +var kafka = require('..'); +var DefaultPartitioner = kafka.DefaultPartitioner; +var RandomPartitioner = kafka.RandomPartitioner; +var CyclicPartitioner = kafka.CyclicPartitioner; +var KeyedPartitioner = kafka.KeyedPartitioner; +var CustomPartitioner = kafka.CustomPartitioner; + +function getPartitions (partitioner, partitions, count) { + var arr = []; + for (var i = 0; i < count; i++) { + arr.push(partitioner.getPartition(partitions)); + } + return arr; +} + +describe('Partitioner', function () { + describe('DefaultPartitioner', function () { + var partitioner = new DefaultPartitioner(); + + describe('#getPartition', function () { + it('should always return the first partition', function () { + var partitions = _.uniq(getPartitions(partitioner, [0, 1], 100)); + partitions.should.have.length(1); + partitions.should.containEql(0); + }); + }); + }); + + describe('RandomPartitioner', function () { + var partitioner = new RandomPartitioner(); + + describe('#getPartition', function () { + it('should return partitions within the existing ones', function () { + var partitions = _.uniq(getPartitions(partitioner, [0, 1], 100)); + partitions.should.have.length(2); + partitions.should.containEql(0); + partitions.should.containEql(1); + }); + }); + }); + + describe('CyclicPartitioner', function () { + var partitioner = new CyclicPartitioner(); + + describe('#getPartition', function () { + it('should return partitions cycling throw the existing ones', function () { + var partitions = getPartitions(partitioner, [0, 1, 2], 6); + partitions.should.have.length(6); + partitions[0].should.equal(0); + partitions[1].should.equal(1); + partitions[2].should.equal(2); + partitions[3].should.equal(0); + partitions[4].should.equal(1); + partitions[5].should.equal(2); + }); + + it('should not modify different partitioners', function () { + var partitioner2 = new CyclicPartitioner(); + var partitions1 = getPartitions(partitioner, [0, 1, 2], 3); + var partitions2 = getPartitions(partitioner2, [0, 1, 2], 3); + partitions1.should.have.length(3); + partitions2.should.have.length(3); + partitions1[0].should.equal(0); + partitions2[0].should.equal(0); + }); + }); + }); + + describe('KeyedPartitioner', function () { + var partitioner = new KeyedPartitioner(); + + describe('#getPartition', function () { + it('should return partitions based on a given key', function () { + var partitions = [partitioner.getPartition([0, 1], '12345'), partitioner.getPartition([0, 1], '123')]; + partitions.should.have.length(2); + partitions[0].should.equal(1); + partitions[1].should.equal(0); + }); + + it('should return partitions based on a given buffer', function () { + var partitions = [partitioner.getPartition([0, 1, 2], Buffer.from([5, 4, 3, 2])), partitioner.getPartition([0, 1, 2], Buffer.from([3, 2, 1, 0]))]; + partitions.should.have.length(2); + partitions[0].should.equal(2); + partitions[1].should.equal(0); + }); + }); + }); + + describe('CustomPartitioner', function () { + function getPartition (partitions, key) { + return partitions[partitions.length - 1]; + } + + var partitioner = new CustomPartitioner(getPartition); + + describe('#getPartition', function () { + it('should always return the last partition', function () { + var partitions = _.uniq(getPartitions(partitioner, [0, 1, 2, 3], 100)); + partitions.should.have.length(1); + partitions.should.containEql(3); + }); + }); + }); +}); diff --git a/node_modules/kafka-node/test/test.producer.js b/node_modules/kafka-node/test/test.producer.js new file mode 100644 index 0000000..04c2bd6 --- /dev/null +++ b/node_modules/kafka-node/test/test.producer.js @@ -0,0 +1,242 @@ +'use strict'; + +var kafka = require('..'); +var Producer = kafka.Producer; +var uuid = require('uuid'); +var Client = kafka.KafkaClient; +var KeyedMessage = kafka.KeyedMessage; +const async = require('async'); + +var client, producer, noAckProducer, producerKeyed; + +[ + { + name: 'PLAINTEXT Producer' + }, + { + name: 'SSL Producer', + sslOptions: { + rejectUnauthorized: false + }, + suiteTimeout: 30000 + } +].forEach(function (testParameters) { + var TOPIC_POSTFIX = '_test_' + uuid.v4(); + var EXISTS_TOPIC_3 = '_exists_3' + TOPIC_POSTFIX; + + var sslOptions = testParameters.sslOptions; + var suiteTimeout = testParameters.suiteTimeout; + var suiteName = testParameters.name; + + const kafkaHost = '127.0.0.1:' + (sslOptions == null ? '9092' : '9093'); + + describe(suiteName, function () { + before(function (done) { + if (suiteTimeout) { + this.timeout(suiteTimeout); + } + client = new Client({ kafkaHost, sslOptions }); + producer = new Producer(client); + noAckProducer = new Producer(client, { requireAcks: 0 }); + producerKeyed = new Producer(client, { partitionerType: Producer.PARTITIONER_TYPES.keyed }); + + async.series( + [ + function (callback) { + producer.once('ready', callback); + }, + function (callback) { + producer.createTopics([EXISTS_TOPIC_3], true, callback); + } + ], + done + ); + }); + + after(function (done) { + producer.close(done); + }); + + describe('#send', function () { + before(function (done) { + // Ensure that first message gets the `0` + producer.send([{ topic: EXISTS_TOPIC_3, messages: '_initial' }], function (err, message) { + message.should.be.ok; + message[EXISTS_TOPIC_3].should.have.property('0', 0); + done(err); + }); + }); + + it('should send message successfully', function (done) { + producer.send([{ topic: EXISTS_TOPIC_3, messages: 'hello kafka' }], function (err, message) { + message.should.be.ok; + message[EXISTS_TOPIC_3]['0'].should.be.above(0); + done(err); + }); + }); + + it('should send buffer message successfully', function (done) { + var message = Buffer.from('hello kafka'); + producer.send([{ topic: EXISTS_TOPIC_3, messages: message }], function (err, message) { + message.should.be.ok; + message[EXISTS_TOPIC_3]['0'].should.be.above(0); + done(err); + }); + }); + + it('should send null message successfully', function (done) { + var message = null; + producer.send([{ topic: EXISTS_TOPIC_3, messages: message }], function (err, message) { + message.should.be.ok; + message[EXISTS_TOPIC_3]['0'].should.be.above(0); + done(err); + }); + }); + + it('should convert none buffer message to string', function (done) { + var message = -1; + producer.send([{ topic: EXISTS_TOPIC_3, messages: message }], function (err, message) { + message.should.be.ok; + message[EXISTS_TOPIC_3]['0'].should.be.above(0); + done(err); + }); + }); + + it('should send Message struct successfully', function (done) { + var message = new KeyedMessage('test-key', 'test-message'); + producer.send([{ topic: EXISTS_TOPIC_3, messages: message }], function (err, message) { + message.should.be.ok; + message[EXISTS_TOPIC_3]['0'].should.be.above(0); + done(err); + }); + }); + + it('should support multi messages in one topic', function (done) { + producer.send([{ topic: EXISTS_TOPIC_3, messages: ['hello kafka', 'hello kafka'] }], function (err, message) { + message.should.be.ok; + message[EXISTS_TOPIC_3]['0'].should.be.above(0); + done(err); + }); + }); + + it('should support snappy compression', function (done) { + producer.send( + [ + { + topic: EXISTS_TOPIC_3, + messages: ['hello kafka', 'hello kafka'], + attributes: 2 + } + ], + function (err, message) { + if (err) return done(err); + message.should.be.ok; + message[EXISTS_TOPIC_3]['0'].should.be.above(0); + done(); + } + ); + }); + + it('should support gzip compression', function (done) { + producer.send( + [ + { + topic: EXISTS_TOPIC_3, + messages: ['hello kafka', 'hello kafka'], + attributes: 1 + } + ], + function (err, message) { + if (err) return done(err); + message.should.be.ok; + message[EXISTS_TOPIC_3]['0'].should.be.above(0); + done(); + } + ); + }); + + it('should send message without ack', function (done) { + noAckProducer.send( + [ + { + topic: EXISTS_TOPIC_3, + messages: 'hello kafka' + } + ], + function (err, message) { + if (err) return done(err); + message.result.should.equal('no ack'); + done(); + } + ); + }); + + it('should send message to specified partition even when producer configured with keyed partitioner', function (done) { + producerKeyed.send([{ key: '12345', partition: 0, topic: EXISTS_TOPIC_3, messages: 'hello kafka' }], function ( + err, + message + ) { + message.should.be.ok; + message[EXISTS_TOPIC_3]['0'].should.be.above(0); + done(err); + }); + }); + + describe('Keyed Partitioner', function () { + const createTopic = require('../docker/createTopic'); + const topicWithTwoPartitions = uuid.v4(); + let client, keyed; + + before(function () { + return createTopic(topicWithTwoPartitions, 2, 1).then(function () { + return new Promise(function (resolve, reject) { + client = new Client({ kafkaHost, sslOptions }); + keyed = new Producer(client, { partitionerType: Producer.PARTITIONER_TYPES.keyed }); + client.once('ready', function () { + client.refreshMetadata([topicWithTwoPartitions], function (error) { + if (error) { + return reject(error); + } + resolve(); + }); + }); + }); + }); + }); + + after(function (done) { + keyed.close(done); + }); + + it('should send message to partition determined by keyed partitioner', function (done) { + keyed.send([{ key: '12345', topic: topicWithTwoPartitions, messages: 'hello kafka' }], function ( + err, + message + ) { + message.should.be.ok; + message[topicWithTwoPartitions].should.have.property('1', 0); + done(err); + }); + }); + }); + }); + + describe('#close', function () { + var client, producer; + + before(function (done) { + client = new Client(); + producer = new Producer(client); + producer.on('ready', done); + }); + + after(function (done) { + producer.close(done); + }); + + it('should close successfully', function (done) { + producer.close(done); + }); + }); + }); +}); diff --git a/node_modules/kafka-node/test/test.producerBatch.js b/node_modules/kafka-node/test/test.producerBatch.js new file mode 100644 index 0000000..c674dc7 --- /dev/null +++ b/node_modules/kafka-node/test/test.producerBatch.js @@ -0,0 +1,182 @@ +'use strict'; + +var sinon = require('sinon'); +var kafka = require('..'); +var Producer = kafka.Producer; +var Client = kafka.KafkaClient; +var async = require('async'); + +var client, producer, batchClient, batchProducer, noAckProducer; + +var TOPIC_POSTFIX = '_test_' + Date.now(); +var EXISTS_TOPIC_4 = '_exists_4' + TOPIC_POSTFIX; +var BATCH_SIZE = 500; +var BATCH_AGE = 300; + +var broker = null; + +// Intermittently fails + +xdescribe('No Ack Producer', function () { + before(function (done) { + async.series( + { + setupClient: function (callback) { + client = new Client(); + batchClient = new Client({ noAckBatchOptions: { noAckBatchSize: BATCH_SIZE, noAckBatchAge: BATCH_AGE } }); + producer = new Producer(client); + batchProducer = new Producer(batchClient); + producer.on('ready', function () { + producer.createTopics([EXISTS_TOPIC_4], true, function (err) { + if (err) return callback(err); + callback(); + }); + broker = Object.keys(client.brokers)[0]; + }); + }, + producerSend: function (callback) { + producer.send([{ topic: EXISTS_TOPIC_4, messages: '_initial 1' }], function (err, message) { + if (err) return callback(err); + message.should.be.ok; + message[EXISTS_TOPIC_4].should.have.property('0', 0); + batchProducer.send([{ topic: EXISTS_TOPIC_4, messages: '_initial 2' }], function (err, message) { + message.should.be.ok; + message[EXISTS_TOPIC_4].should.have.property('0', 1); + callback(err); + }); + }); + // Ensure that first message gets the `0` + } + }, + done + ); + }); + + after(function (done) { + async.each( + [producer, batchProducer], + function (producer, callback) { + producer.close(callback); + }, + done + ); + }); + + describe('with no batch client', function () { + before(function (done) { + noAckProducer = new Producer(client, { requireAcks: 0 }); + done(); + }); + + beforeEach(function () { + this.sendSpy = sinon.spy(client.brokers[broker].socket, 'write'); + }); + + afterEach(function () { + this.sendSpy.restore(); + }); + + it('should send message directly', function (done) { + var self = this; + noAckProducer.send( + [ + { + topic: EXISTS_TOPIC_4, + messages: 'hello kafka no batch' + } + ], + function (err, message) { + if (err) return done(err); + setImmediate(function () { + message.result.should.equal('no ack'); + self.sendSpy.args.length.should.be.equal(1); + self.sendSpy.args[0].toString().should.containEql('hello kafka no batch'); + done(); + }); + } + ); + }); + }); + + describe('with batch client', function () { + before(function (done) { + noAckProducer = new Producer(batchClient, { requireAcks: 0 }); + done(); + }); + + beforeEach(function () { + this.sendSpy = sinon.spy(batchClient.brokers[broker].socket, 'write'); + this.clock = sinon.useFakeTimers(0, 'setTimeout'); + }); + + afterEach(function () { + this.sendSpy.restore(); + this.clock.restore(); + }); + + it('should wait to send message 500 ms', function (done) { + var self = this; + noAckProducer.send( + [ + { + topic: EXISTS_TOPIC_4, + messages: 'hello kafka with batch' + } + ], + function (err, message) { + if (err) return done(err); + setImmediate(function () { + message.result.should.equal('no ack'); + self.sendSpy.args.length.should.be.equal(0); + self.clock.tick(BATCH_AGE - 5); + self.sendSpy.args.length.should.be.equal(0); + self.clock.tick(10); + setImmediate(function () { + self.sendSpy.args.length.should.be.equal(1); + self.sendSpy.args[0].toString().should.containEql('hello kafka with batch'); + done(); + }); + }); + } + ); + }); + + it('should send message once the batch max size is reached', function (done) { + var self = this; + var foo = ''; + for (var i = 0; i < BATCH_SIZE; i++) foo += 'X'; + foo += 'end of message'; + noAckProducer.send( + [ + { + topic: EXISTS_TOPIC_4, + messages: 'hello kafka with batch' + } + ], + function (err, message) { + if (err) return done(err); + message.result.should.equal('no ack'); + self.sendSpy.args.length.should.be.equal(0); + noAckProducer.send( + [ + { + topic: EXISTS_TOPIC_4, + messages: foo + } + ], + function (err, message) { + if (err) return done(err); + setImmediate(function () { + message.result.should.equal('no ack'); + self.sendSpy.args.length.should.be.equal(1); + self.sendSpy.args[0].toString().should.containEql('hello kafka with batch'); + self.sendSpy.args[0].toString().should.containEql('end of message'); + done(); + }); + } + ); + } + ); + }); + }); +}); diff --git a/node_modules/kafka-node/test/test.producerStream.js b/node_modules/kafka-node/test/test.producerStream.js new file mode 100644 index 0000000..129ead1 --- /dev/null +++ b/node_modules/kafka-node/test/test.producerStream.js @@ -0,0 +1,44 @@ +'use strict'; + +const ProducerStream = require('../lib/producerStream'); +const ConsumerGroup = require('../lib/consumerGroup'); +const uuid = require('uuid'); +const createTopic = require('../docker/createTopic'); +const _ = require('lodash'); +const async = require('async'); + +describe('Producer Stream', function () { + let topic; + + before(function () { + topic = uuid.v4(); + return createTopic(topic, 1, 1); + }); + + it('should stream to a topic and verify data was streamed to that topic', function (done) { + const producerStream = new ProducerStream(); + + const messages = [uuid.v4(), uuid.v4()]; + + const consumer = new ConsumerGroup( + { + kafkaHost: '127.0.0.1:9092', + groupId: uuid.v4(), + fromOffset: 'earliest' + }, + topic + ); + + consumer.on('message', function (message) { + _.pull(messages, message.value); + if (_.isEmpty(messages)) { + async.parallel([callback => producerStream.close(callback), callback => consumer.close(callback)], done); + } + }); + + producerStream.write({ + topic: topic, + messages: messages + }); + }); +}); diff --git a/node_modules/kafka-node/test/test.protocol.js b/node_modules/kafka-node/test/test.protocol.js new file mode 100644 index 0000000..1302d3c --- /dev/null +++ b/node_modules/kafka-node/test/test.protocol.js @@ -0,0 +1,40 @@ +'use strict'; + +const versionSupport = require('../lib/protocol/protocolVersions'); +const protocolStruct = require('../lib/protocol/protocol_struct'); +const _ = require('lodash'); + +describe('Protocol', function () { + it('exports correct properties', function () { + versionSupport.should.have.property('apiMap'); + versionSupport.should.have.property('maxSupport'); + versionSupport.should.have.property('baseSupport'); + }); + + describe('verify API map keys', function () { + it('should contain the same keys as request type', function () { + Object.keys(protocolStruct.REQUEST_TYPE).should.be.eql(Object.keys(versionSupport.apiMap)); + Object.keys(protocolStruct.REQUEST_TYPE).should.be.eql(Object.keys(versionSupport.maxSupport)); + Object.keys(protocolStruct.REQUEST_TYPE).should.be.eql(Object.keys(versionSupport.baseSupport)); + }); + + it('should contain different versions of encode/decode functions', function () { + _.forOwn(versionSupport.apiMap, function (value) { + if (value === null) { + return; + } + value.should.be.an.instanceOf(Array); + value.length.should.be.above(0); + for (let requestResponse of value) { + requestResponse.should.have.a.lengthOf(2); + // let encoder = requestResponse[0]; + // encoder.name.should.startWith('encode'); + // encoder.name.should.not.startWith('decode'); + + // encoder.name.should.endWith('Request'); + // encoder.name.should.not.endWith('Response'); + } + }); + }); + }); +}); diff --git a/node_modules/kafka-node/test/test.rebalance.js b/node_modules/kafka-node/test/test.rebalance.js new file mode 100644 index 0000000..d7e7e32 --- /dev/null +++ b/node_modules/kafka-node/test/test.rebalance.js @@ -0,0 +1,238 @@ +'use strict'; + +var kafka = require('..'); +var Client = kafka.KafkaClient; +var Producer = kafka.Producer; +var async = require('async'); +var debug = require('debug')('kafka-node:Test-Rebalance'); +var Childrearer = require('./helpers/Childrearer'); +var uuid = require('uuid'); +var _ = require('lodash'); + +describe('Integrated Reblance', function () { + describe('ConsumerGroup using Kafka Client', function () { + testRebalance('test/helpers/child-cg-kafka-client'); + }); +}); + +function testRebalance (forkPath) { + var producer; + var topic = 'RebalanceTopic'; + var rearer; + var groupId = 'rebal_group'; + + before(function (done) { + if (process.env.TRAVIS) { + return this.skip(); + } + var client = new Client(); + producer = new Producer(client); + client.on('ready', function () { + client.refreshMetadata([topic], function (data) { + // client.topicPartitions[topic].should.be.length(3); + done(); + }); + }); + }); + + beforeEach(function (done) { + rearer = new Childrearer(forkPath); + done(); + }); + + afterEach(function (done) { + debug('killChildren'); + rearer.closeAll(done); + }); + + function sendMessages (messages, done) { + const payload = distributeMessages(messages); + debug('Sending', payload); + producer.send(payload, function (error) { + if (error) { + return done(error); + } + debug('all messages sent'); + }); + } + + function distributeMessages (messages) { + const partitions = [0, 1, 2]; + var index = 0; + var len = partitions.length; + + var partitionBuckets = partitions.map(function (partition) { + return { + topic: topic, + messages: [], + partition: partition + }; + }); + + messages.forEach(function (message) { + partitionBuckets[index++ % len].messages.push(message); + }); + + return partitionBuckets; + } + + function getConsumerVerifier (messages, expectedPartitionsConsumed, expectedConsumersConsuming, done) { + var processedMessages = 0; + var consumedByConsumer = {}; + var verified = _.once(done); + + return function onData (data) { + debug('From child %d %j', this._childNum, data); + topic.should.be.eql(data.message.topic); + if (~messages.indexOf(data.message.value)) { + processedMessages++; + consumedByConsumer[data.id] = true; + } + if (processedMessages >= messages.length) { + var consumedBy = Object.keys(consumedByConsumer); + if (consumedBy.length >= expectedConsumersConsuming) { + verified(); + } else { + verified( + new Error( + 'Received messages but not by the expected ' + + expectedConsumersConsuming + + ' consumers: ' + + JSON.stringify(consumedBy) + ) + ); + } + } + }; + } + + function generateMessages (numberOfMessages, prefix) { + return _.times(numberOfMessages, function () { + return prefix + '-' + uuid.v4(); + }); + } + + it('verify two consumers consuming messages on all partitions', function (done) { + var messages = generateMessages(3, 'verify 2 c'); + var numberOfConsumers = 2; + + var verify = getConsumerVerifier(messages, 3, numberOfConsumers, done); + + rearer.setVerifier(topic, groupId, verify); + rearer.raise(numberOfConsumers, function () { + sendMessages(messages, done); + }); + }); + + it('verify three consumers consuming messages on all partitions', function (done) { + var messages = generateMessages(3, 'verify 3 c'); + var numberOfConsumers = 3; + + var verify = getConsumerVerifier(messages, 3, numberOfConsumers, done); + + rearer.setVerifier(topic, groupId, verify); + rearer.raise(numberOfConsumers); + + sendMessages(messages, done); + }); + + it('verify three of four consumers are consuming messages on all partitions', function (done) { + var messages = generateMessages(3, 'verify 4 c'); + + var verify = getConsumerVerifier(messages, 3, 3, done); + rearer.setVerifier(topic, groupId, verify); + rearer.raise(4); + + sendMessages(messages, done); + }); + + it('verify one consumer consumes all messages on all partitions after one out of the two consumer is killed', function (done) { + var messages = generateMessages(4, 'verify 1 c 1 killed'); + var verify = getConsumerVerifier(messages, 3, 1, done); + + rearer.setVerifier(topic, groupId, verify); + rearer.raise( + 2, + function () { + rearer.kill(1, function () { + sendMessages(messages, done); + }); + }, + 500 + ); + }); + + it('verify two consumer consumes all messages on all partitions after two out of the four consumers are killed right away', function (done) { + var messages = generateMessages(3, 'verify 4 c 2 killed'); + var verify = getConsumerVerifier(messages, 3, 2, done); + + rearer.setVerifier(topic, groupId, verify); + rearer.raise(4, function () { + rearer.kill(2, function () { + sendMessages(messages, done); + }); + }); + }); + + it('verify three consumer consumes all messages on all partitions after one that is unassigned is killed', function (done) { + var messages = generateMessages(3, 'verify 2 c 2 killed'); + var verify = getConsumerVerifier(messages, 3, 2, done); + + rearer.setVerifier(topic, groupId, verify); + + async.series( + [ + function (callback) { + rearer.raise(3, callback); + }, + function (callback) { + setTimeout(callback, 1000); + }, + function (callback) { + rearer.raise(1, callback); + }, + function (callback) { + setTimeout(callback, 1000); + }, + function (callback) { + rearer.killFirst(callback); + } + ], + function () { + sendMessages(messages, done); + } + ); + }); + + it('verify two consumer consumes all messages on all partitions after two out of the four consumers are killed', function (done) { + var messages = generateMessages(3, 'verify 2 c 2 killed'); + var verify = getConsumerVerifier(messages, 3, 2, done); + + rearer.setVerifier(topic, groupId, verify); + rearer.raise( + 4, + function () { + rearer.kill(2, function () { + sendMessages(messages, done); + }); + }, + 500 + ); + }); + + it('verify three consumer consumes all messages on all partitions after three out of the six consumers are killed', function (done) { + var messages = generateMessages(3, 'verify 3 c 3 killed'); + var verify = getConsumerVerifier(messages, 3, 2, done); + + rearer.setVerifier(topic, groupId, verify); + rearer.raise( + 6, + function () { + rearer.kill(3, function () { + sendMessages(messages, done); + }); + }, + 1000 + ); + }); +}