i creating distributed computing server , corresponding client python. server creates daemon pool using multiprocessing waits connections client, either sends data client. client performs operation can take 12 hours. after finishing, client send data back.
because operations can take long, able disconnect server , client sockets process can service more clients. however, have found functions close socket.
server 111 def worker(socket, lock): 122 "workers follow workflow" 123 while true: 124 #accept connection 125 client, address = socket.accept() 126 logger.debug("{u} connected".format(u=address)) 127 client.send("connection established") 128 #receive orders 129 orders = client.recv(1024) 130 if orders == "request work": 131 lock.acquire() 132 jobname = findjob() 133 lock.release() 134 print jobname 135 filenames = getjobfiles(jobname) 136 #send job description 137 numfiles = len(filenames) 138 desc = "sending " + jobname + " " + str(numfiles) + " " 139 blank ='' 140 jobdesc = desc.replace('\n', blank) 141 in filenames: 142 name = i.replace(jobsdir, blank) 143 jobdesc += name 144 jobdesc += ' ' 145 client.send(jobdesc) 146 ready = client.recv(1024) 147 while ready != "ready": 148 ready = client.recv(1024) 149 in filenames: 150 sendfile(client, i) 151 elif orders == 'return data': 152 client.send('ready') 153 #receive information files sent 154 data = client.recv(1024) 155 #interpret data 156 jobname, filenames = parse(data) 157 numfiles = len(filenames) 158 client.send('ready') 159 #receive of files client 160 in range(numfiles): 161 print 'receiving ', filenames[i] 162 recvfile(client, jobname, filenames[i]) 163 #lock , remove job jobs list 164 lock.acquire() 165 removejob(jobname) 166 lock.release() 167 empty = jobsempty() 168 if empty: 169 os.system('python jobs/analyze.py') 170 print 'jobs , analysis finished' 171 sys.exit(0) client 80 s = socket.socket(socket.af_inet, socket.sock_stream) 81 #get local machine name 82 host = socket.gethostname() 83 port = 5007 84 while true: 85 #connect server 86 s.connect((host, port)) 87 #receive confirmation of connection 88 data = s.recv(1024) 89 #request work 90 s.send('request work') 91 #receive string containing file names 92 data = s.recv(1024) 93 if data == 'no new work': break 94 #get files 95 jobname, filenames = parse(data) 96 os.mkdir(jobname) 97 numfiles = len(filenames) 98 print numfiles 99 s.send('ready') 100 in range(numfiles): 101 print 102 print 'receiving ', filenames[i] 103 recvfile(s, filenames[i]) 104 print 'all files received. closing connection' 105 #close socket 106 s.shutdown() 107 #do work 108 os.system('python %s/orders.py'%jobname) 109 #after completing work, initiate connection send files 110 time.sleep(4) 111 s.connect((host, port)) 112 #receive confirmation of connection 113 data = s.recv(1024) 114 #tell server prepare incoming data 115 s.send('return data') 116 data = s.recv(1024) 117 while data != 'ready': 118 data = s.recv(1024) 119 #tell server being sent 120 newfilenames = getjobfiles(jobname) 121 newnumfiles = len(newfilenames) 122 desc = "sending " + jobname + " " + str(numfiles) + " " 123 blank = '' 124 jobdesc = desc.replace('\n', blank) 125 in filenames: 126 jobdesc += name 127 jobdesc += ' ' 128 s.send(jobdesc) 129 #wait server ready 130 ready = s.recv(1024) 131 while ready != 'ready': 132 ready = s.recv(1024) 133 #send files 134 in filenames: 135 sendfile(s, i) 136 s.recv(1024) 137 print 'all files sent' is there way this? i'm sure there has way.
thanks help
Comments
Post a Comment