F
fishboy
Howdy,
Sorry if this is a double post. First try seemed to go into
hyperspace.
I'm working on a personal project. It's going to be a multipart
binary attachment downloader that will search alternate servers for
missing pieces. This is the working code so far. It will walk a
newsgroup and download and decode all the single part attachments.
Just change server,user,password,group at the bottom to something less
virtual.
I'm posting this to get some feedback on my use of generators (and
anything else). It's my first time using them in a non-trivial
program, and I'd like to know if there is more 'Pythonic' way I could
be doing any of this.
My email is obviously fake, so responding here would be great.
Thanks in advance,
David Fisher
#!/usr/bin/env python2.3
#
import asyncore
import socket
import os
import uu
import email
#
class Newser(asyncore.dispatcher):
def __init__(self, host,port,user,password,group):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.connect( (host,port) )
self.buffer = ''
self.user = user
self.password = password
self.group = group
self.n = 0
self.head = ''
self.body = ''
self.inbuffer = ''
self.dataline = ''
self.handleline = self.handleline_gen()
def handle_connect(self):
pass
def handle_close(self):
pass
def writable(self):
return (len(self.buffer) > 0)
def handle_write(self):
print 'sending: ' + self.buffer.strip()
sent = self.send(self.buffer)
self.buffer = self.buffer[sent:]
if self.buffer:
print 'didnt send whole line' #does this ever happen?
print 'didnt send whole line' #just getting my attention
print 'didnt send whole line' #in case it does
def handle_read(self):
self.inbuffer += self.recv(8192)
while 1:
n = self.inbuffer.find('\r\n')
if n > -1:
self.dataline = self.inbuffer[:n+2]
self.inbuffer = self.inbuffer[n+2:]
try:
result = self.handleline.next()
if result == 'OK':
pass # everything is groovy
elif result == 'DONE':
self.del_channel() # group walk is finished
break
else:
print 'something has gone wrong!'
print result
print self.dataline
self.del_channel()
break
except StopIteration:
print 'should never be here'
print 'why did my generator run out?'
print 'why god? why?!'
print self.dataline
self.del_channel()
break
else:
break
def handleline_gen(self):
#
# handshakey stuff
# welcome username password group
# after this is set we'll start the message walk
#
if self.dataline[:3] == '200': # welcome, post ok
print self.dataline.strip()
self.buffer = 'authinfo user ' + self.user + '\r\n'
yield 'OK'
else:
yield 'WTF?! fail welcome? god hates me!'
#
if self.dataline[:3] == '381': # more auth needed
print self.dataline.strip()
self.buffer = 'authinfo pass ' + self.password + '\r\n'
yield 'OK'
else:
yield 'WTF?! fail authinfo user'
#
if self.dataline[:3] == '281': # auth ok, go to town!
print self.dataline.strip()
self.buffer = 'group ' + self.group + '\r\n'
yield 'OK'
else:
yield 'WTF?! fail authinfo pass'
#
if self.dataline[:3] == '211': # group
print self.dataline.strip()
self.buffer = 'next\r\n'
yield 'OK'
else:
yield 'WTF?! fail group'
#
# main state loop
# walk from one message to the next
# issuing HEAD and BODY for each
# never reenter here after we receive '421', no next article
# so we should never issue StopIterator
#
while 1:
#
if self.dataline[:3] == '223': # next
print self.dataline.strip()
self.buffer = 'head\r\n'
yield 'OK'
elif self.dataline[:3] == '421': # err, no next article
yield 'DONE'
else:
yield 'WTF?! fail next'
#
if self.dataline[:3] == '221': # head
print self.dataline.strip()
self.head = ''
yield 'OK'
# XXX what am I going to do if the server explodes
while self.dataline <> '.\r\n':
self.head += self.dataline
yield 'OK'
# XXX parse headers here
# XXX decide whether we want body
self.buffer = 'body\r\n'
yield 'OK'
else:
yield 'WTF?! fail head'
#
if self.dataline[:3] == '222': # body
print self.dataline.strip()
self.body = ''
yield 'OK'
# XXX what am I going to do if the server explodes
while self.dataline <> '.\r\n':
# XXX line-by-line decode here (someday)
self.body += self.dataline
yield 'OK'
self.decode()
self.buffer = 'next\r\n'
yield 'OK'
else:
yield 'WTF?! fail body'
def decode(self):
"""decode message body.
try UU first, just decode body
then mime, decode head+body
save in tempfile if fail"""
tempname = 'temp' + `self.n` + '.txt'
self.n += 1
file(tempname,'wb').write(self.body)
f = file(tempname)
try:
uu.decode(f)
except Exception,v:
print 'uu failed code: ',v
print 'trying MIME'
file(tempname,'wb').write(self.head+self.body)
f = file(tempname)
message = email.message_from_file(f)
for part in message.walk():
print part.get_content_type()
filename = part.get_filename()
if filename:
if not os.path.isfile(filename):
file(filename,'wb').write(part.get_payload(decode=True))
print 'yay! MIME!'
os.remove(tempname)
else:
print "oops, we've already got one"
else:
print 'yay! UU!'
os.remove(tempname)
def main():
mynews = Newser('news.server',119,'fishboy','pass','alt.binaries')
try:
asyncore.loop()
except KeyboardInterrupt:
mynews.del_channel()
print 'yay! I quit!'
if __name__ == '__main__':
main()
Sorry if this is a double post. First try seemed to go into
hyperspace.
I'm working on a personal project. It's going to be a multipart
binary attachment downloader that will search alternate servers for
missing pieces. This is the working code so far. It will walk a
newsgroup and download and decode all the single part attachments.
Just change server,user,password,group at the bottom to something less
virtual.
I'm posting this to get some feedback on my use of generators (and
anything else). It's my first time using them in a non-trivial
program, and I'd like to know if there is more 'Pythonic' way I could
be doing any of this.
My email is obviously fake, so responding here would be great.
Thanks in advance,
David Fisher
#!/usr/bin/env python2.3
#
import asyncore
import socket
import os
import uu
import email
#
class Newser(asyncore.dispatcher):
def __init__(self, host,port,user,password,group):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.connect( (host,port) )
self.buffer = ''
self.user = user
self.password = password
self.group = group
self.n = 0
self.head = ''
self.body = ''
self.inbuffer = ''
self.dataline = ''
self.handleline = self.handleline_gen()
def handle_connect(self):
pass
def handle_close(self):
pass
def writable(self):
return (len(self.buffer) > 0)
def handle_write(self):
print 'sending: ' + self.buffer.strip()
sent = self.send(self.buffer)
self.buffer = self.buffer[sent:]
if self.buffer:
print 'didnt send whole line' #does this ever happen?
print 'didnt send whole line' #just getting my attention
print 'didnt send whole line' #in case it does
def handle_read(self):
self.inbuffer += self.recv(8192)
while 1:
n = self.inbuffer.find('\r\n')
if n > -1:
self.dataline = self.inbuffer[:n+2]
self.inbuffer = self.inbuffer[n+2:]
try:
result = self.handleline.next()
if result == 'OK':
pass # everything is groovy
elif result == 'DONE':
self.del_channel() # group walk is finished
break
else:
print 'something has gone wrong!'
print result
print self.dataline
self.del_channel()
break
except StopIteration:
print 'should never be here'
print 'why did my generator run out?'
print 'why god? why?!'
print self.dataline
self.del_channel()
break
else:
break
def handleline_gen(self):
#
# handshakey stuff
# welcome username password group
# after this is set we'll start the message walk
#
if self.dataline[:3] == '200': # welcome, post ok
print self.dataline.strip()
self.buffer = 'authinfo user ' + self.user + '\r\n'
yield 'OK'
else:
yield 'WTF?! fail welcome? god hates me!'
#
if self.dataline[:3] == '381': # more auth needed
print self.dataline.strip()
self.buffer = 'authinfo pass ' + self.password + '\r\n'
yield 'OK'
else:
yield 'WTF?! fail authinfo user'
#
if self.dataline[:3] == '281': # auth ok, go to town!
print self.dataline.strip()
self.buffer = 'group ' + self.group + '\r\n'
yield 'OK'
else:
yield 'WTF?! fail authinfo pass'
#
if self.dataline[:3] == '211': # group
print self.dataline.strip()
self.buffer = 'next\r\n'
yield 'OK'
else:
yield 'WTF?! fail group'
#
# main state loop
# walk from one message to the next
# issuing HEAD and BODY for each
# never reenter here after we receive '421', no next article
# so we should never issue StopIterator
#
while 1:
#
if self.dataline[:3] == '223': # next
print self.dataline.strip()
self.buffer = 'head\r\n'
yield 'OK'
elif self.dataline[:3] == '421': # err, no next article
yield 'DONE'
else:
yield 'WTF?! fail next'
#
if self.dataline[:3] == '221': # head
print self.dataline.strip()
self.head = ''
yield 'OK'
# XXX what am I going to do if the server explodes
while self.dataline <> '.\r\n':
self.head += self.dataline
yield 'OK'
# XXX parse headers here
# XXX decide whether we want body
self.buffer = 'body\r\n'
yield 'OK'
else:
yield 'WTF?! fail head'
#
if self.dataline[:3] == '222': # body
print self.dataline.strip()
self.body = ''
yield 'OK'
# XXX what am I going to do if the server explodes
while self.dataline <> '.\r\n':
# XXX line-by-line decode here (someday)
self.body += self.dataline
yield 'OK'
self.decode()
self.buffer = 'next\r\n'
yield 'OK'
else:
yield 'WTF?! fail body'
def decode(self):
"""decode message body.
try UU first, just decode body
then mime, decode head+body
save in tempfile if fail"""
tempname = 'temp' + `self.n` + '.txt'
self.n += 1
file(tempname,'wb').write(self.body)
f = file(tempname)
try:
uu.decode(f)
except Exception,v:
print 'uu failed code: ',v
print 'trying MIME'
file(tempname,'wb').write(self.head+self.body)
f = file(tempname)
message = email.message_from_file(f)
for part in message.walk():
print part.get_content_type()
filename = part.get_filename()
if filename:
if not os.path.isfile(filename):
file(filename,'wb').write(part.get_payload(decode=True))
print 'yay! MIME!'
os.remove(tempname)
else:
print "oops, we've already got one"
else:
print 'yay! UU!'
os.remove(tempname)
def main():
mynews = Newser('news.server',119,'fishboy','pass','alt.binaries')
try:
asyncore.loop()
except KeyboardInterrupt:
mynews.del_channel()
print 'yay! I quit!'
if __name__ == '__main__':
main()