2

Using the information from here I can summarize my dataframe using timestamp information.

data <- read.csv("data.csv", header=T)
data$TIMESTAMP <- strptime(data$TIMESTAMP, "%m/%d/%Y %H:%M") # make unambigous
data$TIMESTAMP <- as.POSIXct(data$TIMESTAMP) # plyr/ddply does not seem to like POSIXlt
data$DAY <- as.factor(format(data$TIMESTAMP,'%d'))
data$MONTH <- as.factor(format(data$TIMESTAMP,'%m'))
ddply(data,.(MONTH,DAY),summarise, V1 = mean(P), V2 = max(WS)) # summarize by month by day

However, I only want to use certain hours of the day, for example between 13:00:00 and 16:00:00. I can subset the dataframe (e.g. create a new a dataframe that is a subset of the original df) if I do so before the as.POSIXct line is executed. But, when I try the following sequence:

data.1.4 <- data[format(data$TIMESTAMP, "%H") >= 13 & format(data$TIMESTAMP, "%H") < 17,] # subset the data
data.1.4$TIMESTAMP <- as.POSIXct(data.1.4$TIMESTAMP)
data.1.4$DAY <- as.factor(format(data.1.4$TIMESTAMP,'%d'))
data.1.4$MONTH <- as.factor(format(data.1.4$TIMESTAMP,'%m'))
ddply(data.1.4,.(MONTH,DAY),summarise, V1 = mean(P), V2 = max(WS)) # summarize by month by day

I get the following error Error in eval(expr, envir, enclos) : object 'MONTH' not found. The only difference I can see between data and data.1.4 is that the row.names are 'visible' and non-sequential data.1.4 dataframe.

So, how might I subset or select entries from the dataframe to create a new df that I can then summarize?

# dput of subset of data
data <- structure(list(TIMESTAMP = structure(1:15, .Label = c("1/1/2012 11:00", 
"1/1/2012 12:00", "1/1/2012 13:00", "1/1/2012 14:00", "1/1/2012 15:00", 
"1/2/2012 11:00", "1/2/2012 12:00", "1/2/2012 13:00", "1/2/2012 14:00", 
"1/2/2012 15:00", "4/7/2012 11:00", "4/7/2012 12:00", "4/7/2012 13:00", 
"4/7/2012 14:00", "4/7/2012 15:00"), class = "factor"), P = c(992.4, 
992.4, 992.4, 992.4, 992.4, 992.4, 992.4, 992.4, 992.4, 992.4, 
239, 239, 239, 239, 239), WS = c(4.023, 3.576, 4.023, 6.259, 
4.47, 3.576, 3.576, 2.682, 4.023, 3.576, 2.682, 3.129, 2.682, 
2.235, 2.682), WD = c(212L, 200L, 215L, 213L, 204L, 304L, 276L, 
273L, 307L, 270L, 54L, 24L, 304L, 320L, 321L), AT = c(16.11, 
18.89, 20, 20, 19.44, 10.56, 11.11, 11.67, 12.22, 11.11, 17.22, 
18.33, 19.44, 20.56, 21.11), FT = c(17.22, 22.22, 22.78, 22.78, 
20, 11.11, 15.56, 17.22, 17.78, 15.56, 24.44, 25.56, 29.44, 30.56, 
29.44), H = c(50L, 38L, 38L, 39L, 48L, 24L, 19L, 18L, 16L, 18L, 
23L, 20L, 18L, 17L, 15L), B = c(1029L, 1027L, 1026L, 1024L, 1023L, 
1026L, 1025L, 1024L, 1023L, 1023L, 1034L, 1033L, 1032L, 1031L, 
1030L), FM = c(14.9, 14.4, 14, 13.7, 13.6, 13.1, 12.8, 12.3, 
12, 11.7, 12.8, 12, 11.4, 10.9, 10.4), GD = c(204L, 220L, 227L, 
222L, 216L, 338L, 311L, 326L, 310L, 273L, 62L, 13L, 312L, 272L, 
281L), MG = c(8.047, 9.835, 10.28, 13.41, 11.18, 9.388, 8.941, 
8.494, 9.835, 10.73, 6.706, 7.153, 8.047, 8.047, 7.6), SR = c(522L, 
603L, 604L, 526L, 248L, 569L, 653L, 671L, 616L, 487L, 972L, 1053L, 
1061L, 1002L, 865L), WS2 = c(2.235, 3.576, 4.47, 4.47, 5.364, 
4.023, 2.682, 3.576, 3.576, 4.023, 3.129, 3.129, 3.576, 2.682, 
3.129), WD2 = c(200L, 201L, 206L, 210L, 211L, 319L, 315L, 311L, 
302L, 290L, 49L, 39L, 15L, 348L, 329L)), .Names = c("TIMESTAMP", 
"P", "WS", "WD", "AT", "FT", "H", "B", "FM", "GD", "MG", "SR", 
"WS2", "WD2"), class = "data.frame", row.names = c(NA, -15L))
Community
  • 1
  • 1
nofunsally
  • 2,051
  • 6
  • 35
  • 53
  • Your example code runs fine for me, at least when I use your `dput()` output in place of reading in the csv. No error message about a missing `MONTH` object. – Marius Jan 31 '13 at 03:45
  • You got no error message when using the `data.1.4` dataframe? – nofunsally Jan 31 '13 at 03:51
  • Nope, the `ddply(data.1.4,.(MONTH,DAY),summarise, V1 = mean(P), V2 = max(WS))` runs fine. Maybe clear your workspace and run your code again? – Marius Jan 31 '13 at 04:18
  • @nofunsally, it runs fine for me as well. However, I notice the problem with `plyr` and your code in another post with regard to `POSIXlt`. Can you paste your sessionInfo()? – Arun Jan 31 '13 at 07:18

1 Answers1

3

nofunsally, here's a solution that you might want to consider using data.table. The code is even smaller if you want to take mean of all columns where hour >= 13 and hour <= 15

# changing the name from `TIMESTAMP` to `ts` : too long for me!
names(data)[1] <- "ts"

Step 1: Get the data.table and convert ts to POSIXct

require(data.table)
dt <- data.table(data) # convert to data.table
dt[, ts := as.POSIXct(ts, format="%m/%d/%Y %H:%M")] # convert to POSIXct

Step 2: Get hour, day and month

# get hour, day, month
dt[, `:=`(day=as.POSIXlt(ts)$mday + 1, 
          month=as.POSIXlt(ts)$mon + 1, 
          hour=as.POSIXlt(ts)$hour)]
# set keys for blazing fast access
setkey(dt, "day","month")

step 3: Get all the necessary mean (one-line code)

# the `by` part groups your data.table by day and month
# The group can be accessed by `.SD` and we filter there for hours
# then we select all the columns that we want using `with=FALSE` that makes 
# data.table behave like a data.frame only for that operation
# then we use `lapply` to take the mean of all the columns.
# all this can be done in one line! That's the sweetest part!

dt[, lapply(.SD[hour >= 13 & hour <= 15][, 2:13, with=FALSE], mean), by = "day,month"]

#    day month     P       WS       WD       AT       FT        H        B       FM       GD        MG       SR   WS2
# 1:   2     1 992.4 4.917333 210.6667 19.81333 21.85333 41.66667 1024.333 13.76667 221.6667 11.623333 459.3333 4.768
# 2:   3     1 992.4 3.427000 283.3333 11.66667 16.85333 17.33333 1023.333 12.00000 303.0000  9.686333 591.3333 3.725
# 3:   8     4 239.0 2.533000 315.0000 20.37000 29.81333 16.66667 1031.000 10.90000 288.3333  7.898000 976.0000 3.129

Alternatively, you could first subset for hour and then get mean by selecting the columns:

dt1 <- dt[, .SD[hour >= 13 & hour <= 15], by="day,month"]
dt1[, lapply(.SD, mean), by="day,month", .SDcols=4:17]
Arun
  • 116,683
  • 26
  • 284
  • 387