R使用ddply或聚合 | 码农家园

R use ddply or aggregate

我有一个包含3列的数据框：custId，saleDate，DelivDateTime。

1
2
3
4
5
6
7
8

> head(events22)
custId saleDate DelivDate
1 280356593 2012-11-14 14:04:59 11/14/12 17:29
2 280367076 2012-11-14 17:04:44 11/14/12 20:48
3 280380097 2012-11-14 17:38:34 11/14/12 20:45
4 280380095 2012-11-14 20:45:44 11/14/12 23:59
5 280380095 2012-11-14 20:31:39 11/14/12 23:49
6 280380095 2012-11-14 19:58:32 11/15/12 00:10

这是赔率：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

> dput(events22)
structure(list(custId = c(280356593L, 280367076L, 280380097L,
280380095L, 280380095L, 280380095L, 280364279L, 280364279L, 280398506L,
280336395L, 280364376L, 280368458L, 280368458L, 280368456L, 280368456L,
280364225L, 280391721L, 280353458L, 280387607L, 280387607L),
saleDate = structure(c(1352901899.215, 1352912684.484, 1352914714.971,
1352925944.429, 1352925099.247, 1352923112.636, 1352922476.55,
1352920666.968, 1352915226.534, 1352911135.077, 1352921349.592,
1352911494.975, 1352910529.86, 1352924755.295, 1352907511.476,
1352920108.577, 1352906160.883, 1352905925.134, 1352916810.309,
1352916025.673), class = c("POSIXct","POSIXt"), tzone ="UTC"),
DelivDate = c("11/14/12 17:29","11/14/12 20:48","11/14/12 20:45",
"11/14/12 23:59","11/14/12 23:49","11/15/12 00:10","11/14/12 23:35",
"11/14/12 22:59","11/14/12 20:53","11/14/12 19:52","11/14/12 23:01",
"11/14/12 19:47","11/14/12 19:42","11/14/12 23:31","11/14/12 23:33",
"11/14/12 22:45","11/14/12 18:11","11/14/12 18:12","11/14/12 19:17",
"11/14/12 19:19")), .Names = c("custId","saleDate","DelivDate"
), row.names = c("1","2","3","4","5","6","7","8","9",
"10","11","12","13","14","15","16","17","18","19","20"
), class ="data.frame")

我正在尝试为每个custId查找最新的saleDate的DelivDate。

我可以这样使用plyr :: ddply做到这一点：

1
2
3

dd1 <-ddply(events22, .(custId),.inform = T, function(x){
x[x$saleDate == max(x$saleDate),"DelivDate"]
})

我的问题是，是否有更快的方法来完成此操作，因为ddply方法非常耗时(整个数据集约为40万行)。我已经看过使用aggregate()，但不知道如何获取除我排序依据的值以外的其他值。

有什么建议吗？

编辑：

这是10k行@ 10次迭代的基准结果：

1
2
3
4
5
6

test replications elapsed relative user.self
2 AGG2() 10 5.96 1.000 5.93
1 AGG1() 10 20.87 3.502 20.75
5 DATATABLE() 10 61.32 1 60.31
3 DDPLY() 10 80.04 13.430 79.63
4 DOCALL() 10 90.43 15.173 88.39

EDIT2：
虽然速度最快，但AGG2()无法给出正确的答案。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

> head(agg2)
custId saleDate DelivDate
1 280336395 2012-11-14 16:38:55 11/14/12 19:52
2 280353458 2012-11-14 15:12:05 11/14/12 18:12
3 280356593 2012-11-14 14:04:59 11/14/12 17:29
4 280364225 2012-11-14 19:08:28 11/14/12 22:45
5 280364279 2012-11-14 19:47:56 11/14/12 23:35
6 280364376 2012-11-14 19:29:09 11/14/12 23:01
> agg2 <- AGG2()
> head(agg2)
custId DelivDate
1 280336395 11/14/12 17:29
2 280353458 11/14/12 17:29
3 280356593 11/14/12 17:29
4 280364225 11/14/12 17:29
5 280364279 11/14/12 17:29
6 280364376 11/14/12 17:29
> agg2 <- DDPLY()
> head(agg2)
custId V1
1 280336395 11/14/12 19:52
2 280353458 11/14/12 18:12
3 280356593 11/14/12 17:29
4 280364225 11/14/12 22:45
5 280364279 11/14/12 23:35
6 280364376 11/14/12 23:01

我也将在这里推荐data.table，但是由于您要求使用aggregate解决方案，因此以下是结合了aggregate和merge以获得所有列的解决方案：

1	merge(events22, aggregate(saleDate ~ custId, events22, max))

或者仅aggregate(如果您只需要" custId"和" DelivDate"列)：

1
2
3

aggregate(list(DelivDate = events22$saleDate),
list(custId = events22$custId),
function(x) events22[["DelivDate"]][which.max(x)])

最后，这是使用sqldf的选项：

1
2
3

library(sqldf)
sqldf("select custId, DelivDate, max(saleDate) `saleDate`
from events22 group by custId")

基准测试

我不是基准测试或data.table专家，但令我惊讶的是data.table在这里不是更快。我怀疑在较大的数据集(例如，您的40万行)上，结果会大不相同。无论如何，这是根据@mnel的答案在此处建模的一些基准测试代码，因此您可以对实际数据集进行一些测试以供将来参考。

1	library(rbenchmark)

首先，为要进行基准测试的功能设置功能。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

DDPLY <- function() {
x <- ddply(events22, .(custId), .inform = T,
function(x) {
x[x$saleDate == max(x$saleDate),"DelivDate"]})
}
DATATABLE <- function() { x <- dt[, .SD[which.max(saleDate), ], by = custId] }
AGG1 <- function() {
x <- merge(events22, aggregate(saleDate ~ custId, events22, max)) }
AGG2 <- function() {
x <- aggregate(list(DelivDate = events22$saleDate),
list(custId = events22$custId),
function(x) events22[["DelivDate"]][which.max(x)]) }
SQLDF <- function() {
x <- sqldf("select custId, DelivDate, max(saleDate) `saleDate`
from events22 group by custId") }
DOCALL <- function() {
do.call(rbind,
lapply(split(events22, events22$custId), function(x){
x[which.max(x$saleDate), ]
})
)
}

第二，进行基准测试。

1
2
3
4
5
6
7
8
9

benchmark(DDPLY(), DATATABLE(), AGG1(), AGG2(), SQLDF(), DOCALL(),
order ="elapsed")[1:5]
# test replications elapsed relative user.self
# 4 AGG2() 100 0.285 1.000 0.284
# 3 AGG1() 100 0.891 3.126 0.896
# 6 DOCALL() 100 1.202 4.218 1.204
# 2 DATATABLE() 100 1.251 4.389 1.248
# 1 DDPLY() 100 1.254 4.400 1.252
# 5 SQLDF() 100 2.109 7.400 2.108

相关讨论

这应该很快，但是data.table可能更快：

1
2
3
4
5

do.call(rbind,
lapply(split(events22, events22$custId), function(x){
x[which.max(x$saleDate), ]
})
)

这是一个更快的data.table函数：

1
2
3
4
5

DATATABLE <- function() {
dt <- data.table(events, key=c('custId', 'saleDate'))
dt[, maxrow := 1:.N==.N, by = custId]
return(dt[maxrow==TRUE, list(custId, DelivDate)])
}

请注意，此函数将创建data.table并对数据进行排序，这是您只需执行一次的步骤。如果删除此步骤(作为第一步，也许您有一个多步骤数据处理管道，并且一次创建了data.table)，则该功能的运行速度是其两倍以上。

我还修改了所有先前的函数以返回结果，以便于比较：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

DDPLY <- function() {
return(ddply(events, .(custId), .inform = T,
function(x) {
x[x$saleDate == max(x$saleDate),"DelivDate"]}))
}
AGG1 <- function() {
return(merge(events, aggregate(saleDate ~ custId, events, max)))}

SQLDF <- function() {
return(sqldf("select custId, DelivDate, max(saleDate) `saleDate`
from events group by custId"))}
DOCALL <- function() {
return(do.call(rbind,
lapply(split(events, events$custId), function(x){
x[which.max(x$saleDate), ]
})
))
}

这是1万行的结果，重复10次：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

library(rbenchmark)
library(plyr)
library(data.table)
library(sqldf)
events <- do.call(rbind, lapply(1:500, function(x) events22))
events$custId <- sample(1:nrow(events), nrow(events))

benchmark(a <- DDPLY(), b <- DATATABLE(), c <- AGG1(), d <- SQLDF(),
e <- DOCALL(), order ="elapsed", replications=10)[1:5]

test replications elapsed relative user.self
2 b <- DATATABLE() 10 0.13 1.000 0.13
4 d <- SQLDF() 10 0.42 3.231 0.41
3 c <- AGG1() 10 12.11 93.154 12.03
1 a <- DDPLY() 10 32.17 247.462 32.01
5 e <- DOCALL() 10 56.05 431.154 55.85

由于所有函数均返回其结果，因此我们可以验证它们是否均返回相同的答案：

1
2
3
4
5
6

c <- c[order(c$custId),]
dim(a); dim(b); dim(c); dim(d); dim(e)
all(a$V1==b$DelivDate)
all(a$V1==c$DelivDate)
all(a$V1==d$DelivDate)
all(a$V1==e$DelivDate)

/ Edit：在较小的20行数据集上，data.table仍然是最快的，但是差距较小：

1
2
3
4
5
6

test replications elapsed relative user.self
2 b <- DATATABLE() 100 0.22 1.000 0.22
3 c <- AGG1() 100 0.42 1.909 0.42
5 e <- DOCALL() 100 0.48 2.182 0.49
1 a <- DDPLY() 100 0.55 2.500 0.55
4 d <- SQLDF() 100 1.00 4.545 0.98

/ Edit2：如果从函数中删除data.table创建，则会得到以下结果：

1
2
3
4
5
6
7
8
9
10
11
12
13

dt <- data.table(events, key=c('custId', 'saleDate'))
DATATABLE2 <- function() {
dt[, maxrow := 1:.N==.N, by = custId]
return(dt[maxrow==TRUE, list(custId, DelivDate)])
}
benchmark(a <- DDPLY(), b <- DATATABLE2(), c <- AGG1(), d <- SQLDF(),
e <- DOCALL(), order ="elapsed", replications=10)[1:5]
test replications elapsed relative user.self
2 b <- DATATABLE() 10 0.09 1.000 0.08
4 d <- SQLDF() 10 0.41 4.556 0.39
3 c <- AGG1() 10 11.73 130.333 11.67
1 a <- DDPLY() 10 31.59 351.000 31.50
5 e <- DOCALL() 10 55.05 611.667 54.91