如何從兩個具有通用值的列表創建虛擬矩陣？

Question

在R中，我有多個非常大的IP地址列表（大約140e6）。多個列表之間有許多重疊IP。我想創建一個數據框或數據表，其中包含作爲rowname（沒有重複）的IP地址和列表名稱作爲列和一個0或1表示該IP是否存在於該列表中。

例如，我們有以下兩個列表，兩者之間有一些％相交。

a <- c("192.168.0.1","192.168.0.2","192.168.0.3","192.168.0.4","192.168.0.5","192.168.0.6","192.168.0.7","192.168.0.8","192.168.0.9","192.168.0.10") 
b <- c("192.168.1.1","192.168.1.2","192.168.1.3","192.168.1.4","192.168.0.5","192.168.0.6","192.168.0.7","192.168.0.8","192.168.0.9","192.168.0.10") 

我想是這樣的：

   a b 
192.168.0.1 1 0 
192.168.0.2 1 0 
192.168.0.3 1 0 
192.168.0.4 1 0 
192.168.0.5 1 1 
192.168.0.6 1 1 
192.168.0.7 1 1 
192.168.0.8 1 1 
192.168.0.9 1 1 
192.168.0.10 1 1 
192.168.1.1 0 1 
192.168.1.2 0 1 
192.168.1.3 0 1 
192.168.1.4 0 1 

我一直在使用reshape2，tidyr，model.matrix嘗試，交叉和良好的醇」 for循環。我發現了幾個人從數據框中創建虛擬矩陣的例子，但是沒有使用向量名稱作爲列和值作爲rowname，而不是重複。

Answer 1

一個dplyr解決方案：

df <- data.frame("IP" = unique(c(a,b))) 
df2 <- df%>%mutate(a = ifelse(df$IP %in% a,1,0),b = ifelse(df$IP %in% b,1,0)) 

輸出：

> df2 
      IP a b 
1 192.168.0.1 1 0 
2 192.168.0.2 1 0 
3 192.168.0.3 1 0 
4 192.168.0.4 1 0 
5 192.168.0.5 1 1 
6 192.168.0.6 1 1 
7 192.168.0.7 1 1 
8 192.168.0.8 1 1 
9 192.168.0.9 1 1 
10 192.168.0.10 1 1 
11 192.168.1.1 0 1 
12 192.168.1.2 0 1 
13 192.168.1.3 0 1 
14 192.168.1.4 0 1 

Answer 2

首先，我來介紹2級新的解決方案

與合併的溶液

df1 <- merge(data.frame(ip=a,a=1), data.frame(ip=b,b=1),all=TRUE) %>% 
set_rownames(.,`[`(.,,'ip')) %>% select(-ip) %>% replace(.,is.na(.),0) 

#    a b 
# 192.168.0.1 1 0 
# 192.168.0.10 1 1 
# 192.168.0.2 1 0 
# 192.168.0.3 1 0 
# 192.168.0.4 1 0 
# 192.168.0.5 1 1 
# 192.168.0.6 1 1 
# 192.168.0.7 1 1 
# 192.168.0.8 1 1 
# 192.168.0.9 1 1 
# 192.168.1.1 0 1 
# 192.168.1.2 0 1 
# 192.168.1.3 0 1 
# 192.168.1.4 0 1 

---

而且這裏也有重塑

解決這一個很酷的事情是，它工作時，你有超過2個源矢量：

df2 <- list(data.frame(a),data.frame(b)) %>% 
    lapply(. %>% transform(source = names(.)) %>% rename_("ip" = names(.)[1])) %>% 
    do.call(rbind,.) %>% 
    transform(v=1) %>% 
    reshape(idvar="ip",timevar="source",direction="wide",sep="") %>% 
    replace(.,is.na(.),0) %>% 
    setNames(gsub("v","",colnames(.))) %>% 
    set_rownames(.,`[`(.,,'ip')) %>% select(-ip) 

#    a b 
# 192.168.0.1 1 0 
# 192.168.0.2 1 0 
# 192.168.0.3 1 0 
# 192.168.0.4 1 0 
# 192.168.0.5 1 1 
# 192.168.0.6 1 1 
# 192.168.0.7 1 1 
# 192.168.0.8 1 1 
# 192.168.0.9 1 1 
# 192.168.0.10 1 1 
# 192.168.1.1 0 1 
# 192.168.1.2 0 1 
# 192.168.1.3 0 1 
# 192.168.1.4 0 1 

---

所有解決方案的基準2個載體

讓我們對迄今爲止提供的解決方案進行基準測試。我在使用data.table和從reshape2使用dcast我的第二溶液的變化和spread我的第一溶液的變化添加從tidyR

microbenchmark(
merge = merge(data.frame(ip=a,a=1), data.frame(ip=b,b=1),all=TRUE) %>% 
    set_rownames(.,`[`(.,,'ip')) %>% select(-ip) %>% replace(.,is.na(.),0), 
merge_dt = merge(data.table(ip=a,a=1,key="ip"), data.table(ip=b,b=1,key="ip"),all=TRUE) %>% 
    as.data.frame %>% # to go back to desired output format 
    set_rownames(.,`[`(.,,'ip')) %>% select(-ip) %>% replace(.,is.na(.),0), 
dcast = list(data.frame(a),data.frame(b)) %>% 
    lapply(. %>% transform(source = names(.)) %>% rename_("ip" = names(.)[1])) %>% 
    do.call(rbind,.) %>% 
    transform(v=1) %>% 
    dcast(ip ~ source,value.var="v") %>% 
    replace(.,is.na(.),0) %>% 
    setNames(gsub("v","",colnames(.))) %>% 
    set_rownames(.,`[`(.,,'ip')) %>% select(-ip), 
spread = list(data.frame(a),data.frame(b)) %>% 
    lapply(. %>% transform(source = names(.)) %>% rename_("ip" = names(.)[1])) %>% 
    do.call(rbind,.) %>% 
    transform(v=1) %>% 
    spread(source,v) %>% 
    replace(.,is.na(.),0) %>% 
    setNames(gsub("v","",colnames(.))) %>% 
    set_rownames(.,`[`(.,,'ip')) %>% select(-ip), 
reshape = list(data.frame(a),data.frame(b)) %>% 
    lapply(. %>% transform(source = names(.)) %>% rename_("ip" = names(.)[1])) %>% 
    do.call(rbind,.) %>% 
    transform(v=1) %>% 
    reshape(idvar="ip",timevar="source",direction="wide",sep="") %>% 
    replace(.,is.na(.),0) %>% 
    setNames(gsub("v","",colnames(.))) %>% 
    set_rownames(.,`[`(.,,'ip')) %>% select(-ip), 
akrun = {lvl <- unique(c(a,b));mapply(table, list(a = factor(a, levels = lvl),b = factor(b, levels = lvl)))}, 
p_routh = {df <- data.frame("IP" = unique(c(a,b)));df2 <- df%>%mutate(a = ifelse(df$IP %in% a,1,0),b = ifelse(df$IP %in% b,1,0))}, 
d.b  = {ALL <- unique(c(a,b));data.frame(sapply(list(a = a, b = b), function(x) as.numeric(ALL %in% x)), row.names = ALL)}, 
times = 100 
) 

對於給定的例子：

# Unit: microseconds 
#  expr  min  lq  mean median  uq  max neval 
# merge 2368.754 2670.8205 3866.2288 2942.6280 3685.1415 38459.947 100 
# merge_dt 4220.084 4702.4700 5547.1978 5222.3705 6239.1685 9170.293 100 
# dcast 6153.875 6870.3760 9031.8770 7521.7570 8793.9045 46529.917 100 
# spread 4329.090 4814.6610 6023.5993 5313.3275 6301.9890 38972.416 100 
# reshape 4376.514 5007.1905 5995.1480 5694.1395 6811.4495 8744.180 100 
# akrun 238.893 304.3680 366.0376 327.7265 416.3815 654.744 100 
# p_routh 1013.967 1190.9255 1418.8037 1296.7450 1651.7220 2162.775 100 
#  d.b 133.072 183.8595 228.7220 207.0415 278.1780 417.974 100 

對於更大的例如： 140E6是有點基準，所以我嘗試1E5。我任意選擇a和b之間約50％的重疊。

n <- 1E5 
set.seed(1) 
a <- sample(2*n,n) 
b <- sample(2*n,n) 

，我運行基準10倍

# Unit: milliseconds 
#  expr  min  lq  mean median  uq  max neval 
# merge 582.41885 617.4348 676.40615 651.84618 698.1091 911.8320 10 
# merge_dt 98.72318 100.6648 114.72754 103.57925 119.9722 176.5360 10 
# dcast 267.51729 347.8337 366.85554 360.17472 411.5002 454.1912 10 
# spread 425.26005 447.7959 471.03577 477.02525 490.0484 502.8333 10 
# reshape 697.14005 738.6921 763.31876 751.01547 791.3207 818.0778 10 
# akrun 791.00964 815.5621 838.08296 832.31382 849.5231 923.6849 10 
# p_routh 78.77724 82.8646 98.38296 84.34238 101.7304 151.0339 10 
#  d.b 191.00546 194.5754 209.02133 200.35484 207.1666 279.7900 10 

我們看到將P勞斯的解決方案是最快的2個載體和dcast是最快的通用解決方案。 merge與data.table可能是140E6行中最快的。

---

通用的解決方案

Hopefulle最後編輯：

我設計了2級通用的解決方案基於我最好的那些受限制的，跑他們在大小10E6的3個向量。

merge_dt_gen <- function(...){ 
    args <- as.character(substitute(list(...)))[-1] 
    dts <- args %>% lapply(.%>% data.table(ip=get(.),key="ip")) 
    all_ips <- data.table(ip = unique(c(...)),key="ip") # all_ips <- data.table(ip = unique(c(a,b))) 
    for(dt in dts){ 
    all_ips <- merge(all_ips,dt,all.x = TRUE,by="ip") 
    } 
    all_ips %>% 
    as.data.frame %>% 
    set_rownames(.,`[`(.,,'ip')) %>% 
    select(-ip) %>% 
    setNames(args) %>% 
    replace(.,!is.na(.),1) %>% 
    replace(.,is.na(.),0) 
} 

d_cast_gen <- function(...){ 
    args <- as.character(substitute(list(...)))[-1] 
    args %>% 
    lapply(.%>% data.frame(get(.)) %>% setNames(c("src","ip"))) %>% 
    do.call(rbind,.) %>% 
    transform(v=1) %>% 
    dcast(ip ~ src,value.var="v") %>% 
    replace(.,is.na(.),0) %>% 
    setNames(gsub("v","",colnames(.))) %>% 
    set_rownames(.,`[`(.,,'ip')) %>% select(-ip) 
} 

n <- 10E6 
set.seed(1) 
a <- sample(2*n,n) 
b <- sample(2*n,n) 
d <- sample(unique(a,b),n) 

microbenchmark(
    d_cast_gen = d_cast_gen(a,b,d), 
    merge_dt_gen = merge_dt_gen(a,b,d), 
    times = 1 
) 

# Unit: seconds 
#   expr  min  lq  mean median  uq  max neval 
# d_cast_gen 70.99771 70.99771 70.99771 70.99771 70.99771 70.99771  1 
# merge_dt_gen 47.41809 47.41809 47.41809 47.41809 47.41809 47.41809  1 

merge與data.table是最快

Answer 3

我們可以通過將 'A'， 'B'，以factor與指定爲組合unique元素 'A'， 'B' levels用做這獲得頻率

lvl <- unique(c(a,b)) 
mapply(table, list(a = factor(a, levels = lvl),b = factor(b, levels = lvl))) 
#    a b 
#192.168.0.1 1 0 
#192.168.0.2 1 0 
#192.168.0.3 1 0 
#192.168.0.4 1 0 
#192.168.0.5 1 1 
#192.168.0.6 1 1 
#192.168.0.7 1 1 
#192.168.0.8 1 1 
#192.168.0.9 1 1 
#192.168.0.10 1 1 
#192.168.1.1 0 1 
#192.168.1.2 0 1 
#192.168.1.3 0 1 
#192.168.1.4 0 1