Python分佈式計算（作品）

Question

我正在使用舊線程發佈試圖解決相同問題的新代碼。 什麼構成安全的泡菜？ this?

sock.py

from socket import socket 
from socket import AF_INET 
from socket import SOCK_STREAM 
from socket import gethostbyname 
from socket import gethostname 

class SocketServer: 
    def __init__(self, port): 
    self.sock = socket(AF_INET, SOCK_STREAM) 
    self.port = port 
    def listen(self, data): 
    self.sock.bind(("127.0.0.1", self.port)) 
    self.sock.listen(len(data)) 
    while data: 
     s = self.sock.accept()[0] 
     siz, dat = data.pop() 
     s.send(siz) 
     s.send(dat) 
     s.close() 

class Socket: 
    def __init__(self, host, port): 
    self.sock = socket(AF_INET, SOCK_STREAM) 
    self.sock.connect((host, port)) 
    def recv(self, size): 
    return self.sock.recv(size) 

pack.py

#http://stackoverflow.com/questions/6234586/we-need-to-pickle-any-sort-of-callable 
from marshal import dumps as marshal_dumps 
from pickle import dumps as pickle_dumps 
from struct import pack as struct_pack 

class packer: 
    def __init__(self): 
    self.f = [] 
    def pack(self, what): 
    if type(what) is type(lambda:None): 
     self.f = [] 
     self.f.append(marshal_dumps(what.func_code)) 
     self.f.append(pickle_dumps(what.func_name)) 
     self.f.append(pickle_dumps(what.func_defaults)) 
     self.f.append(pickle_dumps(what.func_closure)) 
     self.f = pickle_dumps(self.f) 
     return (struct_pack('Q', len(self.f)), self.f) 

unpack.py

from types import FunctionType 
from pickle import loads as pickle_loads 
from marshal import loads as marshal_loads 
from struct import unpack as struct_unpack 
from struct import calcsize 

#http://stackoverflow.com/questions/6234586/we-need-to-pickle-any-sort-of-callable 

class unpacker: 
    def __init__(self): 
    self.f = [] 
    self.fcompiled = lambda:None 
    self.sizeofsize = calcsize('Q') 
    def unpack(self, sock): 
    size = struct_unpack('Q', sock.recv(self.sizeofsize))[0] 
    self.f = pickle_loads(sock.recv(size)) 
    a = marshal_loads(self.f[0]) 
    b = globals() ## 
    c = pickle_loads(self.f[1]) 
    d = pickle_loads(self.f[2]) 
    e = pickle_loads(self.f[3]) 
    self.fcompiled = FunctionType(a, b, c, d, e) 
    return self.fcompiled 

test.py

from unpack import unpacker 
from pack import packer 
from sock import SocketServer 
from sock import Socket 
from threading import Thread 
from time import sleep 

count = 2 
port = 4446 

def f(): 
    print 42 

def server(): 
    ss = SocketServer(port) 
    pack = packer() 
    functions = [pack.pack(f) for nothing in range(count)] 
    ss.listen(functions) 

if __name__ == "__main__": 
    Thread(target=server).start() 
    sleep(1) 
    unpack = unpacker() 
    for nothing in range(count): 
    print unpack.unpack(Socket("127.0.0.1", port)) 

輸出：

<function f at 0x12917d0> 
<function f at 0x12915f0> 

Answer 1

ValueError: insecure string pickle當你的醃菜被損壞時引發。你確定你在一個sock.recv()（unpack.py）中接收了整個醃製對象嗎？

編輯：避免這種情況，你可以做任何大小（你Socket類必須支持接收至與緩衝區大小參數（即

class Socket: 
    def recv(self, bufsize): 
     return self.sock.recv(bufsize) 

）被調用）：

import struct 

struct.pack('Q', len(pickled_list)) 
# Send it, and then send the pickled list. 

在接收機的程序：

import struct 

length = struct.unpack('Q', sock.recv(struct.calcsize('Q')))[0] 
pickled_list = sock.recv(length) 

'Q' 是一種unsigned long long。對於其他結構的東西，請參閱the struct module documentation

Answer 2

我不認爲處理對象旨在通過網絡發送。查看multiprocessing/process.py中的第256行。

# We subclass bytes to avoid accidental transmission of auth keys over network. 

聽起來好像有一個很好的理由給我。如果你想分佈式計算，也許你應該看看library designed for that。