python rdma「Hello World」

Question

這個程序是我第一次嘗試Infiniband編程。我使用這個庫：

https://github.com/jgunthorpe/python-rdma

我可能做的非常有趣的事情是錯誤的。我只是試圖發送一個字節序列「Hello，World」到多播地址。

import sys 
import errno 
import unittest 
import mmap 

import rdma 
import rdma.vmad 
import rdma.IBA as IBA 
import rdma.ibverbs as ibv 
import rdma.satransactor 
import rdma.path 
import rdma.vtools 

from rdma.tools import clock_monotonic 
from pyqtgraph.Qt import QtCore, QtGui 


class umad_self_test(): 
    umad = None; 
    tid = 0; 

    def __init__(self): 
     self.umad = None 
     self.tid = 0 

     self.setUp() 

    def setUp(self): 
     self.end_port = rdma.get_end_port(); 
     self.ctx = rdma.get_verbs(self.end_port); 

     #print self.end_port, self.ctx 

    def tearDown(self): 
     self.ctx.close(); 
     self.ctx = None;  

    def test_basic(self): 
     print self.ctx.query_port(); 
     print self.ctx.query_device(); 

     pd = self.ctx.pd(); 
     print pd,repr(pd) 

     cq = self.ctx.cq(100); 
     print cq,repr(cq) 

     try: 
      cq.resize(200); 
     except rdma.SysError as e: 
      if e.errno != errno.ENOSYS: 
       raise 

     comp = self.ctx.comp_channel() 
     print comp,repr(comp) 

     qp = pd.qp(ibv.IBV_QPT_UD, 100, cq, 100, cq) 
     print qp,repr(qp) 
     print qp.query(0xFFFF) 
     mpath = rdma.path.IBPath(self.ctx. end_port, DLID=0xC000, 
           DGID=IBA.GID("ff02::1")) 
     qp.attach_mcast(mpath) 
     qp.detach_mcast(mpath) 

     buf = mmap.mmap(-1,4096); 

     mr = pd.mr(buf,ibv.IBV_ACCESS_LOCAL_WRITE|ibv.IBV_ACCESS_REMOTE_WRITE); 
     print mr,repr(mr) 
     print "MR",mr.addr,mr.length,mr.lkey,mr.rkey 

     ah = pd.ah(self.end_port.sa_path); 
     print ah,repr(ah) 

     srq = pd.srq(); 
     print srq,repr(srq) 
     print srq.query(); 
     srq.modify(100); 

     pool = rdma.vtools.BufferPool(pd,count=100,size=1024); 
     pool.post_recvs(qp,50); 

     buf_idx = pool.pop(); 
     pool.copy_to("Hello, World!", buf_idx); 
     qp.post_send(pool.make_send_wr(buf_idx,pool.size, mpath)); 

if __name__ == '__main__': 
    if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'): 
     umst = umad_self_test() 
     umst.test_basic() 
     umst.tearDown() 

這是輸出我得到：

[idf@node2 python]$ python myverbs.py 
port_attr(state=4, max_mtu=5, active_mtu=5, gid_tbl_len=128L, port_cap_flags=39389290L, max_msg_sz=1073741824L, bad_pkey_cntr=0L, qkey_viol_cntr=0L, pkey_tbl_len=128L, lid=1L, sm_lid=1L, lmc=0L, max_vl_num=3L, sm_sl=0L, subnet_timeout=18L, init_type_reply=0L, active_width=2L, active_speed=4L, phys_state=5L) 
device_attr(fw_ver='2.7.200', node_guid=GUID('0025:90ff:ff1a:0070'), sys_image_guid=GUID('0025:90ff:ff1a:0073'), max_mr_size=18446744073709551615L, page_size_cap=4294966784L, vendor_id=713L, vendor_part_id=26428L, hw_ver=176L, max_qp=163704L, max_qp_wr=16351L, device_cap_flags=8166518L, max_sge=32L, max_sge_rd=0L, max_cq=65408L, max_cqe=4194303L, max_mr=524272L, max_pd=32764L, max_qp_rd_atom=16L, max_ee_rd_atom=0L, max_res_rd_atom=2619264L, max_qp_init_rd_atom=128L, max_ee_init_rd_atom=0L, atomic_cap=1, max_ee=0L, max_rdd=0L, max_mw=0L, max_raw_ipv6_qp=0L, max_raw_ethy_qp=0L, max_mcast_grp=8192L, max_mcast_qp_attach=248L, max_total_mcast_qp_attach=2031616L, max_ah=0L, max_fmr=0L, max_map_per_fmr=8191L, max_srq=65472L, max_srq_wr=16383L, max_srq_sge=31L, max_pkeys=128L, local_ca_ack_delay=15L, phys_port_cnt=1L) 
pd:0:mlx4_0 PD(Context('mlx4_0',fd=3),0x0) 
cq:0:mlx4_0 CQ(Context('mlx4_0',fd=3),0x0) 
comp_channel:5:mlx4_0 CompChannel(Context('mlx4_0',fd=3),5) 
qp:193:pd:0:mlx4_0 QP(PD(Context('mlx4_0',fd=3),0x0),0x0,193,qp_type=IBV_QPT_UD(4)) 
(<rdma.tools.qp_attr object at 0x7f0d29aa24d0>, <rdma.tools.qp_init_attr object at 0x7f0d29aa2550>) 
mr:0:pd:0:mlx4_0 MR(PD(Context('mlx4_0',fd=3),0x0),0x0,0x7f0d34377000,4096,lkey=0xb0042102,rkey=0xb0042102) 
MR 139694687350784 4096 2953060610 2953060610 
ah:6E616269 AH(0x6e616269) 
srq:0:pd:0:mlx4_0 SRQ(PD(Context('mlx4_0',fd=3),0x0),0x0) 
srq_attr(max_wr=127L, max_sge=1L, srq_limit=0L) 
Traceback (most recent call last): 
    File "myverbs.py", line 97, in <module> 
    umst.test_basic() 
    File "myverbs.py", line 92, in test_basic 
    qp.post_send(pool.make_send_wr(buf_idx,pool.size, mpath)); 
    File "ibverbs.pyx", line 1461, in rdma.ibverbs.QP.post_send 
TypeError: an integer is required 
[idf@node2 python]$ 

Answer 1

我相信直接的問題是，你沒有初始化的路徑對象的dqpn（目的地QP號）。根據IBPath docstring，使用未連接的QP時需要目標QP編號。