Skip to content

Latest commit

 

History

History
78 lines (63 loc) · 2.99 KB

README.md

File metadata and controls

78 lines (63 loc) · 2.99 KB

PythonPipeConnection

Create a connection via pipe between to processes or between to python sessions.

##Purpose Python processes need to share data and/or Python users need to share data while processes are running in paralles. The pipeHandler module allows users to connect their data with data pipes while not having to wait or hold up other processes. This proves especially useful in web services such as web2py, django and Flask.

##script to test pipeConnector

    import pipeHandler
    parent_conn,child_conn=pipeHandler.Pipe()  ##create pipe parent and child
    pH1=pipeHandler.pipeConnector()            ##create pipeConnector
    pH1.connectPipe(parent_conn)               ##used parent conn as the pipe connection for pipeConnector
    pH1.recvPipeData()                         ##start receiving thread
    child_conn.send(['hello parent',3,4,5])    ##send sample data string
    pH1.data ##see the data on the other end 
    ##['hello parent',3,4,5]
    proc1=pipeHandler.Process(target=pipeHandler.doSampleStuff,args=(child_conn,))
    proc1.start()
    pH1.data ##see the data on the other end as the function does it's thing
    #['to infinity and beyond', 11365, 22730]
    #['to infinity and beyond', 11495, 22990]
    #....
    #['to infinity and beyond', 11569, 23138]

###to test pickled pipe connector between two shells

    
    # in shell 1
    from pipeHandler import pickledPipeConnector
    from multiprocessing.reduction import reduce_connection
    from multiprocessing import Pipe, current_process
    import pickle

    reader, writer = Pipe()
    pickled_writer = pickle.dumps(reduce_connection(writer))
    passKey=current_process().authkey
    ppC1=pickledPipeConnector(pickled_writer,passKey)
    ppC1.conn=reader
    ppC1.recvPipeData()


    # in shell 2
    from pipeHandler import pickledPipeConnector
    from multiprocessing.reduction import rebuild_connection
    from multiprocessing import Pipe, current_process
    import pickle
    
    #use the passkey and pickled writer from shell 1
    ##here are examples
    passKey='\xe4A-\x18\xd7;&\xdd!5\xa9\xf7\xae\xd5\xd6v\x0c\xb2\x92\x88\x1a\xb3.av\xde\x93s\xd0\x1f!\xf6'
    pickled_writer="(cmultiprocessing.reduction\nrebuild_connection\np0\n((S'/tmp/pymp-_rZWHC/listener-l6qouC'\np1\nI4\nI00\ntp2\nI01\nI01\ntp3\ntp4\n."
    current_process().authkey=passKey   #import passkey
    upw = pickle.loads(pickled_writer)  #import pickled string
    writer = upw[0](upw[1][0],upw[1][1],upw[1][2])
    # writer = rebuild_connection(upw[1][0],upw[1][1],upw[1][2])
    ppC2=pickledPipeConnector(pickled_writer,passKey)
    ppC2.conn=writer
    ppC2.recvPipeData()

    # # now do (almost) any send and recv
    # #in shell 2
    import numpy
    writer.send(['this is data', numpy.arange(10)])


    # #in shell one
    ppC1.data  ##should produce the data
    ['this is data', array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])]
    reader.send('hello back')

    # #in shell two
    ppC2.data ##should produce the message
    'hello back'