Python：如何減少運行時間？

Question

我有一個問題，如何減少運行時間。
我做的代碼是Python。
它需要一個龐大的數據集作爲輸入，處理它，計算並將輸出寫入數組。大多數計算可能非常簡單，如求和。在輸入文件中，大約有1億行和3列。我面臨的問題是運行時間太長。如何減少運行時間？

這裏是我寫的代碼。
我需要從新文件中計算出所有新值（從GenePair到帶有標題的RM_pval）。提前謝謝你。

fi = open ('1.txt') 
fo = open ('2.txt','w') 

import math 
def log(x): 
    return math.log(x) 

from math import sqrt 

import sys 
sys.path.append('/tools/lib/python2.7/site-packages') 
import numpy 
import scipy 
import numpy as np 
from scipy.stats.distributions import norm 

for line in fi.xreadlines(): 
    tmp = line.split('\t') 

    GenePair = tmp[0].strip() 

    PCC_A = float(tmp[1].strip()) 
    PCC_B = float(tmp[2].strip()) 

    ZVAL_A = 0.5 * log((1+PCC_A)/(1-PCC_A)) 
    ZVAL_B = 0.5 * log((1+PCC_B)/(1-PCC_B)) 

    ABS_ZVAL_A = abs(ZVAL_A) 
    ABS_ZVAL_B = abs(ZVAL_B) 

    Var_A = float(1)/float(21-3) #SAMPLESIZE - 3 
    Var_B = float(1)/float(18-3) #SAMPLESIZE - 3 

    WT_A = 1/Var_A #float 
    WT_B = 1/Var_B #float 

    ZVAL_A_X_WT_A = ZVAL_A * WT_A #float 
    ZVAL_B_X_WT_B = ZVAL_B * WT_B #float 

    SumofWT = (WT_A + WT_B) #float 
    SumofZVAL_X_WT = (ZVAL_A_X_WT_A + ZVAL_B_X_WT_B) #float 

    #FIXED MODEL 
    meanES = SumofZVAL_X_WT/SumofWT #float 
    Var = float(1)/SumofWT #float 
    SE = math.sqrt(float(Var)) #float 
    LL = meanES - (1.96 * SE) #float 
    UL = meanES - (1.96 * SE) #float 
    z_score = meanES/SE #float 
    p_val = scipy.stats.norm.sf(z_score) 

    #CAL 
    ES_POWER_X_WT_A = pow(ZVAL_A,2) * WT_A #float 
    ES_POWER_X_WT_B = pow(ZVAL_B,2) * WT_B #float 
    WT_POWER_A = pow(WT_A,2) 
    WT_POWER_B = pow(WT_B,2) 
    SumofES_POWER_X_WT = ES_POWER_X_WT_A + ES_POWER_X_WT_B 
    SumofWT_POWER = WT_POWER_A + WT_POWER_B 

    #COMPUTE TAU 
    tmp_A = ZVAL_A - meanES 
    tmp_B = ZVAL_B - meanES 
    temp = pow(SumofZVAL_X_WT,2) 

    Q = SumofES_POWER_X_WT - (temp /(SumofWT))  
    if PCC_A !=0 or PCC_B !=0: 
     df = 0 
    else: 
     df = 1 

    c = SumofWT - ((pow(SumofWT,2))/SumofWT) 
    if c == 0: 
     tau_square = 0 
    else: 
     tau_square = (Q - df)/c 

    #calculation 
    Var_total_A = Var_A + tau_square 
    Var_total_B = Var_B + tau_square 

    WT_total_A = float(1)/Var_total_A 
    WT_total_B = float(1)/Var_total_B 

    ZVAL_X_WT_total_A = ZVAL_A * WT_total_A 
    ZVAL_X_WT_total_B = ZVAL_B * WT_total_B 

    Sumoftotal_WT = WT_total_A + WT_total_B 
    Sumoftotal_ZVAL_X_WT= ZVAL_X_WT_total_A + ZVAL_X_WT_total_B 


    #RANDOM MODEL 
    RM_meanES = Sumoftotal_ZVAL_X_WT/Sumoftotal_WT 
    RM_Var = float(1)/Sumoftotal_WT 
    RM_SE = math.sqrt(float(RM_Var)) 
    RM_LL = RM_meanES - (1.96 * RM_SE) 
    RM_UL = RM_meanES + (1.96 * RM_SE) 
    RM_z_score = RM_meanES/RM_Var 
    RM_p_val = scipy.stats.norm.sf(RM_z_score) 

Answer 1

肯定做探查的事情，但... 我想唯一的大提速將由於並行發生。如果你要運行像這樣的CPU綁定問題，利用多核心是非常重要的。嘗試把每一行通過一個不同的（線程/進程）。這當然會引發更多問題，例如數據是否需要與輸入文件的順序相同？如果是這樣的話，只需枚舉它並在big_hairy_func上加上第二個變量即可。

這裏是一些樣板代碼上手

筆記：

把xreadlines已經過時，即使它與大文件for line in file:替換它的交易。

fi = open('1.txt') 
fo = open('2.txt','w') 

import math 
def log(x): 
    return math.log(x) 

from math import sqrt 

import multiprocessing as mp 
import sys 
sys.path.append('/tools/lib/python2.7/site-packages') 
import scipy 
import numpy as np 
from scipy.stats.distributions import norm 

def big_hairy_func(linefromfile): 
    <majority of your post here> 
    return <whatever data you were going to write to 'fo'> 

if __name__ == '__main__': 
    pool = mp.Pool(4) #rule of thumb. Replace '4' with the number of cores on your system 
    result = pool.map(big_hairy_func, (input for input in fi.readlines())) 
    <write the result to fo that you haven't posted> 

xreadlines在python 2.3中被棄用，所以該版本我不確定如果生成器函數將工作。如果您對與您的python版本的兼容性有疑問，請告訴我。