減少glmer模型的大小

Question

我是R的新手，我使用glmer來適應幾個二項模型，我只需要它們調用predict即可使用所得到的概率。不過，我有一個非常大的數據集，甚至只是一個模型的大小變得非常大：

> library(pryr) 
> object_size(mod) 
701 MB 

模型的係數的大小相比簡直是小巫見大巫：

> object_size(coef(mod)) 
1.16 MB 

一樣大小的擬合值：

> object_size(fitted(mod)) 
25.6 MB 

首先，我不明白爲什麼模型的對象大小如此之大。它似乎包含用於擬合模型的原始數據框架，但即使這樣也不能說明尺寸。爲什麼這麼巨大？

其次，是否有可能將模型剝離爲只需調用預測的部分？如果是這樣，我將如何去做這件事？我發現一個帖子，這是爲glm在http://blog.yhathq.com/posts/reducing-your-r-memory-footprint-by-7000x.html完成，但似乎glmer模型訪問不同，並有不同的組件。

任何幫助將不勝感激。

編輯：

挖掘到模型的內部：

> object_size(getME(mod, "X")) 
205 MB 
> object_size(getME(mod, "Z")) 
36.9 MB 
> object_size(getME(mod, "Zt")) 
38.4 MB 
> object_size(getME(mod, "Ztlist")) 
41.6 MB 
> object_size(getME(mod, "mmList")) 
38.4 MB 
> object_size(getME(mod, "y")) 
3.2 MB 
> object_size(getME(mod, "mu")) 
3.2 MB 
> object_size(getME(mod, "u")) 
18.4 kB 
> object_size(getME(mod, "b")) 
19.5 kB 
> object_size(getME(mod, "Gp")) 
56 B 
> object_size(getME(mod, "Tp")) 
472 B 
> object_size(getME(mod, "L")) 
15.5 MB 
> object_size(getME(mod, "Lambda")) 
38.1 kB 
> object_size(getME(mod, "Lambdat")) 
38.1 kB 
> object_size(getME(mod, "Lind")) 
9.22 kB 
> object_size(getME(mod, "Tlist")) 
936 B 
> object_size(getME(mod, "A")) 
38.4 MB 
> object_size(getME(mod, "RX")) 
30.3 kB 
> object_size(getME(mod, "RZX")) 
1.05 MB 
> object_size(getME(mod, "sigma")) 
48 B 
> object_size(getME(mod, "flist")) 
4.89 MB 
> object_size(getME(mod, "fixef")) 
4.5 kB 
> object_size(getME(mod, "beta")) 
496 B 
> object_size(getME(mod, "theta")) 
472 B 
> object_size(getME(mod, "ST")) 
936 B 
> object_size(getME(mod, "REML")) 
48 B 
> object_size(getME(mod, "is_REML")) 
48 B 
> object_size(getME(mod, "n_rtrms")) 
48 B 
> object_size(getME(mod, "n_rfacs")) 
48 B 
> object_size(getME(mod, "N")) 
256 B 
> object_size(getME(mod, "n")) 
256 B 
> object_size(getME(mod, "p")) 
256 B 
> object_size(getME(mod, "q")) 
256 B 
> object_size(getME(mod, "p_i")) 
408 B 
> object_size(getME(mod, "l_i")) 
408 B 
> object_size(getME(mod, "q_i")) 
408 B 
> object_size(getME(mod, "mod")) 
48 B 
> object_size(getME(mod, "m_i")) 
424 B 
> object_size(getME(mod, "m")) 
48 B 
> object_size(getME(mod, "cnms")) 
624 B 
> object_size(getME(mod, "devcomp")) 
2.21 kB 
> object_size(getME(mod, "offset")) 
3.2 MB 

> get_obj_size(mod@resp, "RC") 
         [,1] 
family   673355488 
initialize  673355488 
initialize#lmResp 673355488 
ptr    673355488 
resDev   673355488 
updateMu   673355488 
updateWts   673355488 
wrss    673355488 
eta     3196024 
mu     3196024 
n     3196024 
offset    3196024 
sqrtrwt    3196024 
sqrtXwt    3196024 
weights    3196024 
wtres    3196024 
y     3196024 
Ptr      40 
> get_obj_size(mod@pp, "RC") 
        [,1] 
beta   449419408 
initialize 449419408 
initializePtr 449419408 
ldL2   449419408 
ldRX2   449419408 
linPred  449419408 
ptr   449419408 
setTheta  449419408 
sqrL   449419408 
u    449419408 
X    204549128 
V    182171288 
Ut    38448168 
Zt    38448168 
LamtUt   38353248 
Xwts   3196024 
RZX    1047176 
Lambdat   38136 
VtV    26192 
delu    18408 
u0    18408 
Utr    18408 
Lind    9224 
beta0    496 
delb    496 
Vtr     496 
theta    72 
Ptr     40 

Answer 1

發佈作爲一個不完整的答案現在：

library("lme4") 
gm1 <- glmer(cbind(incidence, size - incidence) ~ period + (1 | herd), 
       data = cbpp, family = binomial) 
library("pryr") 
object_size(gm1) ## 505 kB 

繼史蒂夫·沃克的S3/S4/Reference類字典上市提取領域：

get_obj_size <- function(obj,type="S4") { 
    fields <- switch(type, 
        S4=slotNames(obj), 
        RC=ls(obj)) 
    get_field <- switch(type, 
        S4=function(x) slot(obj,x), 
        RC=function(x) obj[[x]]) 
    field_list <- setNames(lapply(fields,get_field),fields) 
    cbind(sort(sapply(field_list,object_size),decreasing=TRUE)) 
} 
get_obj_size(gm1) 
##   [,1] 
## resp 356620 ## 'response module' 
## pp  355420 ## 'predictor module' 
## frame  6640 
## optinfo 1748 
## devcomp 1424 
## call  1244 
## flist  1232 
## cnms  224 
## u   152 
## beta  56 
## Gp   32 
## lower  32 
## theta  32 

它將值得進一步挖掘響應和預測模塊，看看有什麼/有什麼大，有些信息將被存儲在環境這些組件​​

。例如，我認爲，這通常是相同的尺寸低於組件的整體轉換實際上不是獨立的，而是具有相同的環境...

get_obj_size(gm1@resp,"RC") 
##      [,1] 
## initialize  356620 
## initialize#lmResp 356620 
## ptr    356620 
## resDev   356620 
## setOffset   356620 
## updateMu   356620 
## updateWts   356620 
## wrss    356620 
## family    26016 
## eta     472 
## mu     472 
## n     472 
## offset    472 
## sqrtrwt    472 
## sqrtXwt    472 
## weights    472 
## wtres    472 
## y     472 
## Ptr     20 

另一個查看存儲哪些組件的方法是使用object_size(getME(model,component))並重復通過eval(formals(getME)$name)列出的組件;這與信息在內部存儲的方式不太精確，但會讓您瞭解需要多少空間來容納（例如）固定效應或隨機效應模型矩陣。

我從事這方面的更多一些，並有部分解決，但還有存儲，我似乎無法找到/修剪掉的正確很多（注，這需要在Github上的最新版本lme4：我不得不稍微修改predict函數以減弱對內部結構的依賴）。

glmer_chop <- function(object) { 
    newobj <- object 
    newobj@frame <- model.frame(object)[0,] 
    newobj@pp <- with(object@pp, 
        new("merPredD", 
         Lambdat=Lambdat, 
         Lind=Lind, 
         theta=theta, 
         u=u,u0=u0, 
         n=nrow(X), 
         X=matrix(1,nrow=nrow(X)), 
         Zt=Zt)) ## .sparseDiagonal(n,shape="g"))) 
    newobj@resp <- new("glmResp",family=binomial(),y=numeric(0)) 
    return(newobj) 
} 
get_obj_size(environment(fm2@pp$initialize),"RC") 
fm1 <- glmer(use ~ urban+age+livch+(1|district), Contraception, binomial) 
object_size(Contraception) ## 133 kB 
object_size(fm1) ## 1.05 MB 
object_size(fm2 <- glmer_chop(fm1)) ## 699 kB 
get_obj_size(fm2)   ## 'pp' is 547200 bytes 
get_obj_size(fm2@pp,"RC") ## 'initialize' object is 547200 
saveRDS(fm2,file="tmp.rds") 
fm2 <- readRDS("tmp.rds") 
object_size(fm2) ## 796 kB 
rm(fm1) 
pp <- predict(fm2,newdata=Contraception) 
object_size(fm2) ## still 796K; no sharing 

最後注意到compare_size(fm2)證實，這裏的大多數信息都存儲在環境中，而不是在對象本身（但我不知道怎麼compare_size/object.size處理引用類...）

Answer 2

您對存儲空間或內存的關注？如果是關於存儲的話，其中一個選擇就是嵌入調用，以便在生成預測的代碼中估計模型，這樣您就不會實際存儲模型對象。喜歡的東西：

predictions <- predict(glmer(y ~ x, family = binomial), type = "response")