1

I am trying to retrieve historical events within a certain time range using IoT Foundation's rest API . But if I specify optional start/end parameter of the API call, IoT Foundation doesn't recognize them as expected and returns the events within unexpected time range.
I wonder if I used the API incorrectly or it is an IoT Foundation's bug.
Following is the code(test_hist.js) which retrieves events within last 24 hours.
(My credential parameters are replaced with *s.)

/* IBM Internet of Things Foundation instance specific parameters */
var httpHost = '******.internetofthings.ibmcloud.com';
var authOptions= {
    'user': '*******************', // apikey
    'password': '******************' // apiToken
};
var deviceId = '******************';

var Client = require('node-rest-client').Client;
var restClient = new Client(authOptions);
var endMs   = new Date().valueOf();
var startMs = endMs - 24 * 60 * 60 * 1000;
console.error('DEBUG: start=' + startMs + ' : ' + new Date(startMs));
console.error('DEBUG: end  =' + endMs   + ' : ' + new Date(endMs));
var params = {};
if (process.argv.indexOf('-s') === -1) params.start = startMs; // -s option to omit start parameter
if (process.argv.indexOf('-e') === -1) params.end = endMs; // -e option to omit end parameter
var verbose = (process.argv.indexOf('-v') >= 0); // -v option to output events to stdout.
doGet(params);

function doGet(params, allEvents) {
    var url = 'https://' + httpHost + '/api/v0002/historian/types/HARemoteControl/devices/' + deviceId;
    var args = { parameters: params };
    var req = restClient.get(url, args, function(data, response) {
        if (response.statusCode != 200) {
            console.error(response.statusMessage);
        } else {
            var events = data.events;
            var eventSize = events.length;
            allEvents = !allEvents ? events : allEvents.concat(events);
            var getMore = false;
            if ('bookmark' in data) {
                if (!('_bookmark' in params)) console.error('DEBUG: bookmark=' + data.bookmark);
                params._bookmark = data.bookmark;
                if (0 < eventSize && startMs < events[eventSize-1].timestamp.$date) {
                    doGet(params, allEvents);
                    getMore = true;
                }
            }
            if (eventSize > 0) {
                var lastTime = new Date(events[0].timestamp.$date);
                var firstTime = new Date(events[eventSize-1].timestamp.$date);
                console.error('DEBUG: ' + firstTime + ' ... ' + lastTime + ' events.length=' + eventSize + ('bookmark' in data ? ' more...' : ''));
            } else {
                console.error('DEBUG: events.length=0' + ('bookmark' in data ? ' more...' : ''));
            }
            if (verbose && !getMore) console.log( allEvents );
        }
    });
    if (!allEvents) console.error('DEBUG: path=' + req.options.path);
}

Result:
When I ran this code for the first time, the API returned 24 hours of events, as expected. 

    $ node test_hist.js 
    DEBUG: start=1445993284029 : Wed Oct 28 2015 09:48:04 GMT+0900 (JST)
    DEBUG: end  =1446079684029 : Thu Oct 29 2015 09:48:04 GMT+0900 (JST)
    DEBUG: path=/api/v0002/historian/types/HARemoteControl/devices/HARemoteControl001?start=1445993284029&end=1446079684029
    DEBUG: bookmark=1:iotHistorianSessionId=AnLL7ocgqFvkmyqAKXERIiNCBPvZttRNy7%2bIP7Vi8xJJCQq56QVBE0cSkQMI1W1L; HttpOnly
    DEBUG: Thu Oct 29 2015 08:09:03 GMT+0900 (JST) ... Thu Oct 29 2015 09:48:03 GMT+0900 (JST) events.length=100 more...
    DEBUG: Thu Oct 29 2015 06:29:03 GMT+0900 (JST) ... Thu Oct 29 2015 08:08:03 GMT+0900 (JST) events.length=100 more...
    DEBUG: Thu Oct 29 2015 04:49:03 GMT+0900 (JST) ... Thu Oct 29 2015 06:28:08 GMT+0900 (JST) events.length=100 more...
    DEBUG: Thu Oct 29 2015 03:09:03 GMT+0900 (JST) ... Thu Oct 29 2015 04:48:05 GMT+0900 (JST) events.length=100 more...
    DEBUG: Thu Oct 29 2015 01:29:03 GMT+0900 (JST) ... Thu Oct 29 2015 03:08:03 GMT+0900 (JST) events.length=100 more...
    DEBUG: Wed Oct 28 2015 23:49:03 GMT+0900 (JST) ... Thu Oct 29 2015 01:28:03 GMT+0900 (JST) events.length=100 more...
    DEBUG: Wed Oct 28 2015 22:09:08 GMT+0900 (JST) ... Wed Oct 28 2015 23:48:03 GMT+0900 (JST) events.length=100 more...
    DEBUG: Wed Oct 28 2015 20:39:03 GMT+0900 (JST) ... Wed Oct 28 2015 22:08:03 GMT+0900 (JST) events.length=100 more...
    DEBUG: Wed Oct 28 2015 18:59:03 GMT+0900 (JST) ... Wed Oct 28 2015 20:38:05 GMT+0900 (JST) events.length=100 more...
    DEBUG: Wed Oct 28 2015 17:19:02 GMT+0900 (JST) ... Wed Oct 28 2015 18:58:05 GMT+0900 (JST) events.length=100 more...
    DEBUG: Wed Oct 28 2015 15:39:02 GMT+0900 (JST) ... Wed Oct 28 2015 17:18:08 GMT+0900 (JST) events.length=100 more...
    DEBUG: Wed Oct 28 2015 14:02:12 GMT+0900 (JST) ... Wed Oct 28 2015 15:38:02 GMT+0900 (JST) events.length=100 more...
    DEBUG: Wed Oct 28 2015 12:22:02 GMT+0900 (JST) ... Wed Oct 28 2015 14:01:02 GMT+0900 (JST) events.length=100 more...
    DEBUG: Wed Oct 28 2015 10:44:02 GMT+0900 (JST) ... Wed Oct 28 2015 12:21:05 GMT+0900 (JST) events.length=100 more...
    DEBUG: Wed Oct 28 2015 09:49:02 GMT+0900 (JST) ... Wed Oct 28 2015 10:43:02 GMT+0900 (JST) events.length=55
    $

But when I ran the same code later, the API returned unexpected time period of events.

  • 11:45 of yesterday is specified as start parameter but the API returned the older events(11:16...) than 11:45.
  • 11:45 (of today) is specified as end parameter but the API didn't return events between 10:23 and 11:45 but older than 10:23.

as follows.

    $ node test_hist.js 
    DEBUG: start=1446000327271 : Wed Oct 28 2015 11:45:27 GMT+0900 (JST)
    DEBUG: end  =1446086727271 : Thu Oct 29 2015 11:45:27 GMT+0900 (JST)
    DEBUG: path=/api/v0002/historian/types/HARemoteControl/devices/HARemoteControl001?start=1446000327271&end=1446086727271
    DEBUG: bookmark=1:iotHistorianSessionId=vbazIN0SDWrJTaep06zTipWS1TqbPMv/9m8EEumw5uuL6Gj6FiWl5z9POLeB5xLv; HttpOnly
    DEBUG: Thu Oct 29 2015 08:43:03 GMT+0900 (JST) ... Thu Oct 29 2015 10:22:03 GMT+0900 (JST) events.length=100 more...
    DEBUG: Thu Oct 29 2015 07:03:03 GMT+0900 (JST) ... Thu Oct 29 2015 08:42:13 GMT+0900 (JST) events.length=100 more...
    DEBUG: Thu Oct 29 2015 05:23:05 GMT+0900 (JST) ... Thu Oct 29 2015 07:02:06 GMT+0900 (JST) events.length=100 more...
    DEBUG: Thu Oct 29 2015 03:43:08 GMT+0900 (JST) ... Thu Oct 29 2015 05:22:05 GMT+0900 (JST) events.length=100 more...
    DEBUG: Thu Oct 29 2015 02:03:08 GMT+0900 (JST) ... Thu Oct 29 2015 03:42:03 GMT+0900 (JST) events.length=100 more...
    DEBUG: Thu Oct 29 2015 00:23:03 GMT+0900 (JST) ... Thu Oct 29 2015 02:02:03 GMT+0900 (JST) events.length=100 more...
    DEBUG: Wed Oct 28 2015 22:43:03 GMT+0900 (JST) ... Thu Oct 29 2015 00:22:03 GMT+0900 (JST) events.length=100 more...
    DEBUG: Wed Oct 28 2015 21:13:08 GMT+0900 (JST) ... Wed Oct 28 2015 22:42:13 GMT+0900 (JST) events.length=100 more...
    DEBUG: Wed Oct 28 2015 19:33:03 GMT+0900 (JST) ... Wed Oct 28 2015 21:12:03 GMT+0900 (JST) events.length=100 more...
    DEBUG: Wed Oct 28 2015 17:53:03 GMT+0900 (JST) ... Wed Oct 28 2015 19:32:03 GMT+0900 (JST) events.length=100 more...
    DEBUG: Wed Oct 28 2015 16:13:02 GMT+0900 (JST) ... Wed Oct 28 2015 17:52:03 GMT+0900 (JST) events.length=100 more...
    DEBUG: Wed Oct 28 2015 14:33:02 GMT+0900 (JST) ... Wed Oct 28 2015 16:12:05 GMT+0900 (JST) events.length=100 more...
    DEBUG: Wed Oct 28 2015 12:56:02 GMT+0900 (JST) ... Wed Oct 28 2015 14:32:02 GMT+0900 (JST) events.length=100 more...
    DEBUG: Wed Oct 28 2015 11:16:02 GMT+0900 (JST) ... Wed Oct 28 2015 12:55:02 GMT+0900 (JST) events.length=100 more...
    $

Nodes:

  • The events are sent from my RaspberryPi periodically (every 1 minutes) and sometime sporadically. Therefor 1440+ events are expected to be returned with this code.

  • After this symptom occurred and if start and end parameters are omitted, the API returned the last 100 events, as expected. Which means IoT Foundation has events between 10:23 and 11:45 but did not return them when end parameter is specified.

    $ node test_hist.js -s -e
DEBUG: start=1446000368159 : Wed Oct 28 2015 11:46:08 GMT+0900 (JST)
DEBUG: end  =1446086768159 : Thu Oct 29 2015 11:46:08 GMT+0900 (JST)
DEBUG: path=/api/v0002/historian/types/HARemoteControl/devices/HARemoteControl001
DEBUG: Thu Oct 29 2015 10:07:06 GMT+0900 (JST) ... Thu Oct 29 2015 11:46:03 GMT+0900 (JST) events.length=100
$

  • Above 3 execution results are of a Bluemix VM (Ubuntu 14.4) and between the normal and abnormal results, I ran the same code(test_hist.js) on a local Windows PC at about 10:22AM. So it seems to me that this API memorizes the start/end parameters in some circumstance and keeps replacing start/end parameters of following API calls with that memorized values.

Yuhji Yamashita
  • 11
  • 3

1 Answers1

0

I am having a similar problem, data is being returned truncated & out of date/time order and also probably with missing data. You may check the page bellow where I dump data from 2015-01-01 until 2015-12-31 in the order it is provided by IoTF into a Google Table from the following AJAX query:

/api/v0002/historian/types/6c84fb90-12c4-11e1-840d-7b25c5ee775a/devices/SX.084.045.224.012?evt_type=report:1&start=1420070400000&end=1451520000000

Result: http://www.sensormax.com.br/8/

macedo123
  • 84
  • 1
  • 2
  • It seems there may be another problem but the original problem which is Bluemix historian API does not honor start/end parameter has been resolved, according to: [link](http://status.internetofthings.ibmcloud.com/2015/12/01/historical-event-retrieval-issues-are-now-resolved/). I confirmed this fix with my above test code(test_hist.js). – Yuhji Yamashita Dec 18 '15 at 07:10