Creating a Fast&Simple Container for Sending Messages to a Topic in Oracle Event Hub Cloud Service (aka OEHCS, which is a Kafka cluster) and Deploying it to Kubernetes Cluster

var sleep = require('system-sleep');

const oehcs_connect_url = process.env.OEHCS_CONNECTURL

const topic_name = process.env.TOPIC_NAME

const num_partitions = process.env.NUM_PARTITIONS

const message = process.env.MESSAGE

var kafka = require('kafka-node'),

HighLevelProducer = kafka.HighLevelProducer,

client = new kafka.KafkaClient({kafkaHost: oehcs_connect_url}),

producer = new HighLevelProducer(client);

var i = 0;

while (i >= 0 ){

var payloads = [{ topic: topic_name, messages: message , partition: i}];

//producer.on('ready', function () {

producer.send(payloads, function (err, data) {

console.log(data);

});

// });

i = i + 1;

if (i > num_partitions -1){

i = 0;

sleep( 1 );

}

}

FROM oraclelinux:7-slim
WORKDIR /app
ADD . /app
RUN curl --silent --location https://rpm.nodesource.com/setup_10.x | bash -
RUN yum -y install nodejs npm
CMD ["node","producer-direct.js"]

Create the deployment and after 10 minutes take a look to production messages ratio:

kubectl apply -f my.yaml

More or less 400/second…

Scale the deployment and take a look to new production ratios:

kubectl scale deployment oehcsnodeproducer-direct --replicas=2

Around 8000 messages/second!

Now add 9 partitions to the topic and take a look to new ratios:

With 2 pods running and 10 partitions we are producing around 10K messages per second! As you can see partitioning improves performance!

Let’s double the number of pods and see new ratios:

kubectl scale deployment oehcsnodeproducer-direct --replicas=4

And now 18K/second!

That’s all folks!

Enjoy 😉