How can I get highcharter to represent a forecast object?

Question

This is a follow-on to this question.

I am trying to get the pipeline given in that question to accept a forecast object as input:

Again, using this data:

> dput(t)
structure(c(2, 2, 267822980, 325286564, 66697091, 239352431, 
94380295, 1, 126621669, 158555699, 32951026, 23, 108000151, 132505189, 
29587564, 120381505, 25106680, 117506099, 22868767, 115940080, 
22878163, 119286731, 22881061), .Dim = c(23L, 1L), index = structure(c(1490990400, 
1490994000, 1490997600, 1491001200, 1491004800, 1491008400, 1491012000, 
1491026400, 1491033600, 1491037200, 1491040800, 1491058800, 1491062400, 
1491066000, 1491069600, 1491073200, 1491076800, 1491109200, 1491112800, 
1491120000, 1491123600, 1491156000, 1491159600), tzone = "US/Mountain", tclass = c("POSIXct", 
"POSIXt")), class = c("xts", "zoo"), .indexCLASS = c("POSIXct", 
"POSIXt"), tclass = c("POSIXct", "POSIXt"), .indexTZ = "US/Mountain", tzone = "US/Mountain", .CLASS = "double", .Dimnames = list(
    NULL, "count"))

I use

highchart(type = 'stock') %>% 
    hc_add_series(t) %>% 
    hc_xAxis(type = 'datetime')

To create

But if I follow this same recipe using

require("forecast")
t.arima <- auto.arima(t)
x <- forecast(t.arima, level = c(95, 80))

highchart(type = 'stock') %>% 
     hc_add_series(x) %>%
     hc_xAxis(type = 'datetime')

I get this error:

Error in as.Date.ts(.) : unable to convert ts time to Date class

How can I show the forecast series along with the historical? I've seen this in the documentation, but don't understand why I'd be getting this error.

JS CONSOLE OUTPUT FOR JK: DF DATA AFTER RE-INDEXING:

dput(df)
structure(list(Index = structure(c(1490968800, 1490972400, 1490976000, 
1490979600, 1490983200, 1490986800, 1490990400, 1491004800, 1491012000, 
1491015600, 1491019200, 1491037200, 1491040800, 1491044400, 1491048000, 
1491051600, 1491055200, 1491087600, 1491091200, 1491098400, 1491102000, 
1491134400, 1491138000, 1491217200, 1491220800, 1491224400, 1491228000, 
1491231600, 1491235200, 1491238800, 1491242400, 1491246000, 1491249600, 
1491253200, 1491256800, 1491260400, 1491264000, 1491267600), class = c("POSIXct", 
"POSIXt")), Data = c(2, 2, 259465771, 315866206, 64582553, 233440220, 
91918347, 1, 126563786, 158555699, 32951026, 23, 108000151, 132505189, 
29587564, 120381505, 25106680, 117506099, 22868767, 115898351, 
22878163, 119285747, 22881061, 157925588, 32447780, 223096830, 
281656273, 45406684, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA), 
    Fitted = c(102170573.857143, 102170573.857143, 102170573.857143, 
    102170573.857143, 102170573.857143, 102170573.857143, 102170573.857143, 
    102170573.857143, 102170573.857143, 102170573.857143, 102170573.857143, 
    102170573.857143, 102170573.857143, 102170573.857143, 102170573.857143, 
    102170573.857143, 102170573.857143, 102170573.857143, 102170573.857143, 
    102170573.857143, 102170573.857143, 102170573.857143, 102170573.857143, 
    102170573.857143, 102170573.857143, 102170573.857143, 102170573.857143, 
    102170573.857143, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA), 
    `Point Forecast` = c(NA, NA, NA, NA, NA, NA, NA, NA, NA, 
    NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, 
    NA, NA, NA, NA, 102170573.857143, 102170573.857143, 102170573.857143, 
    102170573.857143, 102170573.857143, 102170573.857143, 102170573.857143, 
    102170573.857143, 102170573.857143, 102170573.857143), `Lo 80` = c(NA, 
    NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, 
    NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, -16003477.5789723, 
    -16003477.5789723, -16003477.5789723, -16003477.5789723, 
    -16003477.5789723, -16003477.5789723, -16003477.5789723, 
    -16003477.5789723, -16003477.5789723, -16003477.5789723), 
    `Hi 80` = c(NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, 
    NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, 
    NA, 220344625.293258, 220344625.293258, 220344625.293258, 
    220344625.293258, 220344625.293258, 220344625.293258, 220344625.293258, 
    220344625.293258, 220344625.293258, 220344625.293258), `Lo 95` = c(NA, 
    NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, 
    NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, -78561041.5917782, 
    -78561041.5917782, -78561041.5917782, -78561041.5917782, 
    -78561041.5917782, -78561041.5917782, -78561041.5917782, 
    -78561041.5917782, -78561041.5917782, -78561041.5917782), 
    `Hi 95` = c(NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, 
    NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, 
    NA, 282902189.306064, 282902189.306064, 282902189.306064, 
    282902189.306064, 282902189.306064, 282902189.306064, 282902189.306064, 
    282902189.306064, 282902189.306064, 282902189.306064)), .Names = c("Index", 
"Data", "Fitted", "Point Forecast", "Lo 80", "Hi 80", "Lo 95", 
"Hi 95"), row.names = c(NA, -38L), class = "data.frame")

score 2 · Accepted Answer · answered Apr 03 '17 at 16:17

2

Not sure this is due to the irregular time series.

Anyway, ggfortify:::fortify.forecast is your friend. Why? Because fortify (try to) transform all the R object in data frames. So:

library(highcharter)
library(forecast)
t.arima <- auto.arima(t)
x <- forecast(t, level = c(95, 80))

library(highcharter)
library(ggplot2)
library(ggfortify)
#> 
#> Attaching package: 'ggfortify'
#> The following object is masked from 'package:forecast':
#> 
#>     gglagplot
class(x)
#> [1] "forecast"

df <- fortify(x)
head(df)
#>   Index      Data    Fitted Point Forecast Lo 80 Hi 80 Lo 95 Hi 95
#> 1     1         2 140658844             NA    NA    NA    NA    NA
#> 2  3601         2 121734145             NA    NA    NA    NA    NA
#> 3  7201 267822980 105355638             NA    NA    NA    NA    NA
#> 4 10801 325286564 127214522             NA    NA    NA    NA    NA
#> 5 14401  66697091 153863779             NA    NA    NA    NA    NA
#> 6 18001 239352431 142136089             NA    NA    NA    NA    NA

Now you can:

highchart(type = "stock") %>% 
  hc_add_series(df, "line", hcaes(Index, Data), name = "Original") %>% 
  hc_add_series(df, "line", hcaes(Index, Fitted), name = "Fitted") %>%
  hc_add_series(df, "line", hcaes(Index, `Point Forecast`), name = "Forecast") %>% 
  hc_add_series(df, "arearange", hcaes(Index, low = `Lo 80`, high = `Hi 80`), name = "Interval")

As you can see, fortify can't detect the real time too. So you need to transform the Index in the time what you want.

answered Apr 03 '17 at 16:17

jbkunst

3,009
1
22
28

So if I reformat the index with `df$Index <- c(zoo::index(t)+7614000, (max(zoo::index(t))+7614000+3600*1:10))` in order to line it up with `2017-03-31` and the 10h forecast front edge, and I rerun this code, the graph that comes back in blank. – d8aninja Apr 03 '17 at 22:12
Open the widget in chrome, press F12 and check the javascript console – jbkunst Apr 04 '17 at 02:10
Okay, I used `df$Index <- c(zoo::index(t), (max(zoo::index(t)) + 3600*1:10))` and I added a snap of the js console to the question. looks to be an empty `svg.highcharts-root` object. – d8aninja Apr 04 '17 at 04:21
Is it a problem with the NAs? I added the current state of `df` to the question, as well – d8aninja Apr 04 '17 at 04:26
It's not an issue with the NAs, I replaced them all with 1s and it still isn't rendering... – d8aninja Apr 04 '17 at 14:09
I've got this "sort of" working with `type = 'line'` but as you can see I lose that beautiful `arearange` geom this way :( Added to the question under "4/4 Edit" – d8aninja Apr 04 '17 at 14:54
It won' let me paste this code in there for some reason. Try: http://www.r-fiddle.org/#/fiddle?id=pT43BsTZ&version=1 – d8aninja Apr 04 '17 at 15:21
Use `datetime_to_timestamp` function, this convert R date time values to numeric date time values which highcharts can interpret `df$Index <- datetime_to_timestamp(c(zoo::index(t), (max(zoo::index(t)) + 3600*1:10)))` – jbkunst Apr 04 '17 at 17:47
`df <- xts::as.xts(select(df, -Index), df$Index)` will no longer work with that class of Index: `Error in xts(x, order.by = order.by, frequency = frequency, ...) : order.by requires an appropriate time-based object` – d8aninja Apr 04 '17 at 18:08

score 2 · Answer 2 · answered Jun 26 '18 at 04:55

The error

Error in as.Date.ts(.) : unable to convert ts time to Date class

is due to the fact that you have a ts object with a frequency that is not covered by the function as.Date.ts(.). When we see what this function does, this is what we get:

function (x, offset = 0, ...) 
{
    time.x <- unclass(time(x)) + offset
    if (frequency(x) == 1) 
        as.Date(paste(time.x, 1, 1, sep = "-"))
    else if (frequency(x) == 4) 
        as.Date(paste((time.x + 0.001)%/%1, 3 * (cycle(x) - 1) + 
            1, 1, sep = "-"))
    else if (frequency(x) == 12) 
        as.Date(paste((time.x + 0.001)%/%1, cycle(x), 1, sep = "-"))
    else stop("unable to convert ts time to Date class")
}

This function considers only 3 values for the frequency of a ts object: 1, 4, or 12. When we take a look at the frequency of your object x, we see that its frequency = 0.000277777777777778, so when highcharter calls the function using the ts objects in x it stops and gives you that error.

We have two options on how to "fix" it:

Transform t into a ts object (instead of a xts object) with frequency = 1 before running auto.arima and forecast;
After running auto.arima and forecast, we can create an index for the future dates and transform the ts objects in x into xts objects with the correct index.

I said "fix" because these solutions are not perfect, as we will see.

Option 1

t <- structure(
  c(2, 2, 267822980, 325286564, 66697091, 239352431,
    94380295, 1, 126621669, 158555699, 32951026, 23, 
    108000151, 132505189, 29587564, 120381505, 25106680,
    117506099, 22868767, 115940080, 22878163, 119286731, 
    22881061), 
  .Dim = c(23L, 1L), 
  index = structure(c(1490990400, 1490994000, 1490997600, 
                      1491001200, 1491004800, 1491008400, 
                      1491012000, 1491026400, 1491033600, 
                      1491037200, 1491040800, 1491058800, 
                      1491062400, 1491066000, 1491069600, 
                      1491073200, 1491076800, 1491109200, 
                      1491112800, 1491120000, 1491123600, 
                      1491156000, 1491159600), 
                    tzone = "US/Mountain", 
                    tclass = c("POSIXct","POSIXt")), 
  class = c("xts", "zoo"), 
  .indexCLASS = c("POSIXct","POSIXt"), 
  tclass = c("POSIXct", "POSIXt"), 
  .indexTZ = "US/Mountain", 
  tzone = "US/Mountain", 
  .CLASS = "double", 
  .Dimnames = list(NULL, "count"))


require("forecast")
library(highcharter)


# SOLUTION 1
t.tmp <- ts(t, start=1, end = length(t))
t.arima.1 <- auto.arima(t.tmp)
x.1 <- forecast(t.arima.1, level = c(95, 80))

highchart(type = 'stock') %>% 
  hc_add_series(x.1) %>%
  hc_add_series(x.1$x, name = "Original") %>% 
  hc_add_series(x.1$fitted, name = "Fitted")

The problem with this approach is that we lose the dates (axis, tooltip, etc.).

Option 2, 1st try: Hourly Forecasts

I tried to create an hourly index for the future values, but for some reason Highcharter moves the intervals to the left (or there's some problem with the dates that I can't see/figure out).

Option 2, 2nd try: Daily Forecasts

When I changed it to a daily index for the future values it worked, but it's weird since we have hourly observations and the forecast part of our plot shows "daily forecasts".

Here is the full code:

t <- structure(
  c(2, 2, 267822980, 325286564, 66697091, 239352431,
    94380295, 1, 126621669, 158555699, 32951026, 23, 
    108000151, 132505189, 29587564, 120381505, 25106680,
    117506099, 22868767, 115940080, 22878163, 119286731, 
    22881061), 
  .Dim = c(23L, 1L), 
  index = structure(c(1490990400, 1490994000, 1490997600, 
                      1491001200, 1491004800, 1491008400, 
                      1491012000, 1491026400, 1491033600, 
                      1491037200, 1491040800, 1491058800, 
                      1491062400, 1491066000, 1491069600, 
                      1491073200, 1491076800, 1491109200, 
                      1491112800, 1491120000, 1491123600, 
                      1491156000, 1491159600), 
                    tzone = "US/Mountain", 
                    tclass = c("POSIXct","POSIXt")), 
  class = c("xts", "zoo"), 
  .indexCLASS = c("POSIXct","POSIXt"), 
  tclass = c("POSIXct", "POSIXt"), 
  .indexTZ = "US/Mountain", 
  tzone = "US/Mountain", 
  .CLASS = "double", 
  .Dimnames = list(NULL, "count"))

require("forecast")
library(highcharter)
library(xts)

t.arima <- auto.arima(t)
x <- forecast(t.arima, level = c(95, 80))

# Problem 

## Time from 'forecast'
time.x <- time(x$mean) # ts variable
time.x # see that frequency = 0.000277777777777778

## Original time
time.t <- time(t) # POSIXct variable, use as.ts to see frequency
as.ts(time.t) # frequency = 1

## Try to transform back to formatted date
as.POSIXct(as.double(time.t), tz = "US/Mountain", origin = "1970-01-01")
as.POSIXct(as.double(time.x), tz = "US/Mountain", origin = "1970-01-01")

#--------------------------------------------------------#

# SOLUTION 1
t.tmp <- ts(t, start=1, end = length(t))
t.arima.1 <- auto.arima(t.tmp)
x.1 <- forecast(t.arima.1, level = c(95, 80))

highchart(type = 'stock') %>% 
  hc_add_series(x.1) %>%
  hc_add_series(x.1$x, name = "Original") %>% 
  hc_add_series(x.1$fitted, name = "Fitted")

#------------------------------------------------------#

# SOLUTION 2 - With correct dates but wrong plot

## Create new forecast variable
x.2 <- forecast(t.arima.1, level = c(95, 80))

## Take forecast length
forecast.length <- length(time.x)

### Create New Forecast dates (HOUR)
### Since I don't know the exact forecast times, I'll add one HOUR
### for each obs starting from the last date in the original dataset 

last.date <- time.t[length(time.t)]

new.forecast.time.hour <- as.POSIXct(last.date) + c((1:forecast.length)*3600)

## Insert date back

x.2$mean  <- xts(x.1$mean, order.by = new.forecast.time.hour)
x.2$lower <- xts(x.1$lower, order.by = new.forecast.time.hour)
x.2$upper <- xts(x.1$upper, order.by = new.forecast.time.hour)

### Original Data
x.2$x <- xts(x.1$x, order.by = time.t)

### Fitted
x.2$fitted <- xts(x.1$fitted, order.by = time.t)

# Plot forecasts with correct date

highchart(type = 'stock') %>% 
  hc_add_series(x.2) %>%
  hc_add_series(x.2$x, name = "Original") %>% 
  hc_add_series(x.2$fitted, name = "Fitted") %>%
  hc_xAxis(type = 'datetime')

#------------------------------------------------------#

# SOLUTION 3 - Correct plot but only for daily forecasts

## Create new forecast variable
x.3 <- forecast(t.arima.1, level = c(95, 80))

## Take forecast length
forecast.length <- length(time.x)

### Create New Forecast dates (DAY)
### Since I don't know the exact forecast times, I'll add one DAY
### for each obs starting from the last date in the original dataset 

last.date <- time.t[length(time.t)]

new.forecast.time.day <- as.POSIXct(last.date) + c((1:forecast.length)*3600*24)
## Add change from as.POSIXct to as.Date
new.forecast.time.day <- as.Date(new.forecast.time.day)

## Insert date back

x.3$mean  <- xts(x.1$mean, order.by = new.forecast.time.day)
x.3$lower <- xts(x.1$lower, order.by = new.forecast.time.day)
x.3$upper <- xts(x.1$upper, order.by = new.forecast.time.day)

### Original Data
x.3$x <- xts(x.1$x, order.by = time.t)

### Fitted
x.3$fitted <- xts(x.1$fitted, order.by = time.t)

# Plot forecasts with correct date

highchart(type = 'stock') %>% 
  hc_add_series(x.3) %>%
  hc_add_series(x.3$x, name = "Original") %>% 
  hc_add_series(x.3$fitted, name = "Fitted") %>%
  hc_xAxis(type = 'datetime')

One other thing: the fitted values on my plots differ from the fitted values on jbkunst's plot because he used forecast directly on t, not on t.arima (just a typo, I believe). This way, my forecasts are based on an Arima model, while his are based on an ETS model.

How can I get highcharter to represent a forecast object?

2 Answers2

Option 1

Option 2, 1st try: Hourly Forecasts

Option 2, 2nd try: Daily Forecasts

Linked