The container uses 4 environment variables, you can find a container already built for you here
SOURCE CODE OF THE PRODUCER
var sleep = require('system-sleep');
const oehcs_connect_url = process.env.OEHCS_CONNECTURL
const topic_name = process.env.TOPIC_NAME
const num_partitions = process.env.NUM_PARTITIONS
const message = process.env.MESSAGE
var kafka = require('kafka-node'),
HighLevelProducer = kafka.HighLevelProducer,
client = new kafka.KafkaClient({kafkaHost: oehcs_connect_url}),
producer = new HighLevelProducer(client);
var i = 0;
while (i >= 0 ){
var payloads = [{ topic: topic_name, messages: message , partition: i}];
//producer.on('ready', function () {
producer.send(payloads, function (err, data) {
console.log(data);
});
// });
i = i + 1;
if (i > num_partitions -1){
i = 0;
sleep( 1 );
}
}
THE DOCKERFILE
FROM oraclelinux:7-slim WORKDIR /app ADD . /app RUN curl --silent --location https://rpm.nodesource.com/setup_10.x | bash - RUN yum -y install nodejs npm CMD ["node","producer-direct.js"]
THE YAML FOR K8S DEPLOYMENT
apiVersion: apps/v1
kind: Deployment
metadata:
name: oehcsnodeproducer-direct
labels:
app: oehcsnodeproducer-direct
spec:
replicas: 1
selector:
matchLabels:
app: oehcsnodeproducer-direct
strategy:
type: Recreate
template:
metadata:
labels:
app: oehcsnodeproducer-direct
spec:
containers:
- image: javiermugueta/oehcsnodeproducer-direct
env:
- name: OEHCS_CONNECTURL
value: "<ip1>:6667,<ip2>:6667,..."
- name: TOPIC_NAME
value: "R1"
- name: NUM_PARTITIONS
value: "10"
- name: MESSAGE
value: "{'put here what you want'}"
name: oehcsnodeproducer-direct
TEST IT AND SEE WHAT HAPPENS
Create the deployment and after 10 minutes take a look to production messages ratio:
kubectl apply -f my.yaml
More or less 400/second…
Scale the deployment and take a look to new production ratios:
kubectl scale deployment oehcsnodeproducer-direct --replicas=2
Around 8000 messages/second!
Now add 9 partitions to the topic and take a look to new ratios:
With 2 pods running and 10 partitions we are producing around 10K messages per second! As you can see partitioning improves performance!
Let’s double the number of pods and see new ratios:
kubectl scale deployment oehcsnodeproducer-direct --replicas=4
And now 18K/second!
That’s all folks!
Enjoy 😉
[…] messages to R1 topic in origin cluster the way you can. See this post for […]
LikeLike