I am trying to visualize streaming data using Google PubSub, Node js, and Google App Engine. What I am doing is simply:
- Push a stream of messages to PubSub topic every 1 second (using a python script)
- Create a subscription to that topic
- Create a simple web app which listens to the subscription, parse each incoming message, and display them to browser in the form of live bar chart (using node js, socket.io, and fusionchart)
I followed this tutorial : https://www.fusioncharts.com/blog/visualize-real-time-data-socket-io-charts/ , the only difference is that I used PubSub instead of PubNub, by also following this documentation : https://cloud.google.com/pubsub/docs/quickstart-client-libraries#pubsub-client-libraries-nodejs for the 'receive message' part.
It actually works, that when I published 10 messages every 1 second, the chart will display exactly 10 messages every 1 second in real time. This is how the chart looks like when I run my app:
The problem is that it only works the very first time I run the app and open the page. When I refresh the page and push another 10 new messages, it took longer for the data to show up. When it finally appears, only 5 out of 10 messages are displayed. If I try to refresh the same page, there will be less and less data displayed or even none of it appears at some point.
I keep track of each incoming and outgoing messages through my console, and seems like everything is okay. 10 messages pushed, 10 received. It's only for some reason, less and less displayed after each refresh.
Here is my app.js code:
var express = require('express');
var app = require('express')();
var http = require('http').Server(app);
//creates a new socket.io instance attached to the http server.
var io = require('socket.io')(http);
// Imports the Google Cloud client library
const PubSub = require('@google-cloud/pubsub');
// Your Google Cloud Platform project ID
const projectId = 'myprojecthere';
//Provide the absolute path to the dist directory.
app.use(express.static(__dirname + '/dist'));
//On get request send 'index.html' page as a response.
app.get('/', function(req, res) {
res.sendFile(__dirname +'/index.html');
});
//Whenever someone connects this gets executed
//original : connection
io.on('connection', function (socket) {
console.log(`Enter io connection`);
console.log(' %s sockets connected', io.engine.clientsCount)
// Instantiates a client
const pubsub = new PubSub({
projectId: projectId,
key: """censored"""
});
var strData;
/**
* TODO(developer): Uncomment the following lines to run the sample.
* https://cloud.google.com/pubsub/docs/pull#pubsub-pull-messages-async-nodejs
*/
const subscriptionName = 'testing_subscription';
const topicName = 'testing';
const timeout = 50;
// References an existing subscription
//var topic = pubsub.topic(topicName)
const subscription = pubsub.subscription(subscriptionName);
//Function to format time and date
function formatDatetime (TimeStamp){
var formatted = (TimeStamp.getHours()) + ':' + (TimeStamp.getMinutes()) + ':' + (TimeStamp.getSeconds()) + ':' + (TimeStamp.getMilliseconds());
return formatted;
}
// Create an event handler to handle messages
let messageCount = 0;
const messageHandler = message => {
console.log(`Received message: ${message.id}`);
console.log(`\tData: ${message.data}`);
console.log(`\tAttributes: ${message.attributes}`);
var obj = JSON.parse(message.data);
console.log(`\tTimeStamp: ${obj.messages.timestamp}`);
console.log(`\tAmount: ${obj.messages.amount}`);
messageCount += 1;
console.log(`Message count : ${messageCount}`);
message.ack();
console.log(`Message Acknowledged`);
// This doesn't ack the message, but allows more messages to be retrieved
// if your limit was hit or if you don't want to ack the message.
// message.nack();
// Get creation timestamp
var x = new Date(obj.messages.timestamp);
// Time formatting for x-axis in chart
var formatted = formatDatetime(x);
var Count = obj.messages.amount;
console.log(`Extracting Timestamp: ${formatted}`);
console.log(`Counts : ${Count}`);
strData = {"label": formatted,
"value": Count
}
socket.emit('news', strData);
console.log(``);
};
// Listen for new messages until timeout is hit
subscription.on(`message`, messageHandler);
setTimeout(() => {
console.log(`Enter timeout`);
//subscription.removeListener('message', messageHandler);
console.log(`0 message(s) received.`);
var x = new Date();
var formatted = formatDatetime(x);
var Count = 0;
console.log(`Extracting Timestamp: ${formatted}`)
strData = {"label": formatted,
"value": Count
}
console.log(`strData : ${strData}`)
console.log(``);
socket.emit('news', strData);
}, timeout);
//other handling
if ( typeof strData == 'undefined') {
console.log(`Something else happened`)
var x = new Date();
var formatted = formatDatetime(x);
console.log(`Extracting Timestamp: ${formatted}`)
strData = {"label": formatted,
"value": 9
}
socket.emit('news', strData);
}
console.log(`strData : ${strData}`);
console.log(``);
});
//server listening on port 8080
http.listen(8080, function() {
console.log('listening on *:8080');
});Here is the code I used to display the chart:
/*globals io */
var FusionCharts = require("fusioncharts");
require("fusioncharts/fusioncharts.charts")(FusionCharts);
require("fusioncharts/fusioncharts.widgets")(FusionCharts);
var socket = io();
var transactionChart = new FusionCharts({
id: "mychart",
type: 'realtimecolumn',
width: '700',
height: '350',
dataFormat: 'json',
dataSource: {
"chart": {
"caption": "Streaming Data Visualization",
"subCaption": "Testing",
"yaxismaxvalue": "10",
"numdisplaysets": "10",
"yAxisName":"Quantity",
"labeldisplay": "rotate",
"showLegend":"0",
"showValues": "0",
"numbersuffix": "Kg",
"showlabels": "1",
/*This parameter lets you set whether you want the latest value (received from server) to be displayed on the chart or not*/
"showRealTimeValue": "0",
/*For this parameter, you can specify the number of seconds after which the chart will look for new data. This process will happen continuously - i.e., if you specify 5 seconds here, the chart will look for new data every 5 seconds*/
"refreshInterval":".1",
/*If you want the chart to keep polling for new data every x seconds and queue it, you can specify that x seconds as updateInterval. This helps you poll at different intervals and then draw at another interval (specified as refreshInterval)*/
"updateInterval":".1",
"yAxisNamePadding":"10",
//Cosmetics
"paletteColors" : "#0075c2,#1aaf5d",
"baseFontColor" : "#333333",
"baseFont" : "Helvetica Neue,Arial",
"captionFontSize" : "14",
"subcaptionFontSize" : "14",
"subcaptionFontBold" : "0",
"showBorder" : "0",
"bgColor" : "#ffffff",
"showShadow" : "0",
"canvasBgColor" : "#ffffff",
"canvasBorderAlpha" : "0",
"divlineAlpha" : "100",
"divlineColor" : "#999999",
"divlineThickness" : "1",
"divLineIsDashed" : "1",
"divLineDashLen" : "1",
"divLineGapLen" : "1",
"usePlotGradientColor" : "0",
"showplotborder" : "0",
"valueFontColor" : "#ffffff",
"placeValuesInside" : "1",
"rotateValues" : "1",
"showXAxisLine" : "1",
"xAxisLineThickness" : "1",
"xAxisLineColor" : "#999999",
"showAlternateHGridColor" : "0",
"legendBgAlpha" : "0",
"legendBorderAlpha" : "0",
"legendShadow" : "0",
"legendItemFontSize" : "10",
"legendItemFontColor" : "#666666"
},
"categories": [
{
"category": [
{ "label": "Start" }
]
}
],
"dataset": [
{
"seriesname": "",
"alpha": "100",
"data": [
{ "value": "3" }
]
}
]
}
}).render("chart-container");
//On connection with socket, will start receiving the data
socket.connect('http://localhost:8080/');
socket.on('news', function (data) {
function updateData() {
//Converting the fetched data in FusionCharts format
var strData = "&label=" + data.label + "&value=" + data.value;
//feeding the data to the real time chart
FusionCharts.items.mychart.feedData(strData);
}
//calling the update method
updateData();
});And here is my index.html code:
<!DOCTYPE html>
<html>
<head>
<title>Hello world</title>
<script src="/socket.io/socket.io.js"></script>
</head>
<body>
<div id="chart-container">FusionCharts will render here</div>
<script src="bundle.js"></script>
</body>
</html>I am still very new to Javascript and have never been work on web app before. There might be important knowledge that I missed on how things work. But I do have some suspicion, though I am not sure.
Could it possibly because a new socket connection is made everytime I refesh the page, and the missing messages are actually received by previous connection (hence it's not displayed)?
Few solutions I have tried and still didn't work: Node.js Socket.io page refresh multiple connections
Anyone can help me with this?
