I have setup multiple threads and a queue...which is working pretty
good. But I have one other issue...I have a new thread (since it is
different issue) here:
http://groups.google.com/group/comp.lang.python/browse_frm/thread/ec81d8982d1a0130
if you get chance, would you mind checking that out.
I haven't sent you my bill for the current effort yet; let's see,
about three hours of effort over two days... $150 <G> {That ignores that
a real consulting rate would have to include the overhead for both sides
of Social Security and various medical insurance coverages, not just my
base employed rate <$G$>}
What I came up with (at work, strangely -- very unproductive day)...
'Tis a tad long as I stuffed a mass of comment lines into it (not to
mention the test run capture includes a copy of the file from a "type"
command)...
-=-=-=-=-=-=-=-
import subprocess
import os
import threading
import Queue
class Shell(object):
def __init__(self, command):
# flag for shutdown
self.die = False
self.timeOut = 1.0
# queue for buffering "lines"
self.lineQueue = Queue.Queue()
# list for collecting characters read from subprocess
# allows for character reader to collect characters
# while line reader processes a prior batch
self.charList = []
# event to signal that, at least, one character is
# available. event is used to allow time-out wait()
self.charEvent = threading.Event()
# event to signal that something has been written to the
# subprocess, and that the line reader should start
# collecting stuff (this is used to avoid multiple "prompt"
# timeouts from falsely affecting the user
self.lineEvent = threading.Event()
# lock to wrap access to self.charList
self.charLock = threading.Lock()
# create subprocess object with supplied command
self.process = subprocess.Popen(command,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE)
# initialize, and start the character reader
# thread -- this is the thread that actually reads the
# subprocess output stream, one character at a time
self.charThread = threading.Thread(target=self.readChar)
self.charThread.start()
# initialize, and start the line reader thread
# -- this is the thread that processes collected characters
# OR on a timeout of the event, queues up a prompt and
# resets the event flags to block until more data is expected
self.lineThread = threading.Thread(target=self.readLine)
self.lineThread.start()
# set the line event so the line reader knows it should read
up
# to the next time-out/prompt
self.lineEvent.set()
def readChar(self):
while not self.die:
# read one character at a time from subprocess; blocks
ch = self.process.stdout.read(1)
# get the lock to access the character list, append it
self.charLock.acquire()
self.charList.append(ch)
# signal the line reader that, at least one, character
# is in the list
self.charEvent.set()
# let the line reader access the character list
self.charLock.release()
def readLine(self):
lineChar = []
while not self.die:
# block until told that the subprocess has been commanded
# and thereby should be producing some sort of output
self.lineEvent.wait()
# block for up to one second waiting for the character
# reader to produce some sort of output
self.charEvent.wait(self.timeOut)
# get the lock allowing access to the character list
self.charLock.acquire()
# clear the character reader event so the next pass
# will block(time-out) if no more data
self.charEvent.clear()
# save the character list locally so the character reader
# can add more stuff to it (after clearing it)
tCharList = self.charList
self.charList = []
# release the character list lock so reader can touch it
self.charLock.release()
# if the list is not empty, we have some data
if tCharList:
# process each character looking for a new line
character
# note: no filtering is done, all characters will be
# returned to user
for ch in tCharList:
lineChar.append(ch)
# on a new line, collect all preceding and make a
# string out of them, queue the string
if ch == "\n":
self.lineQueue.put("".join(lineChar))
lineChar = []
else:
# time out, assume prompt stage, queue partial line
self.lineQueue.put("".join(lineChar))
lineChar = []
# queue None as signal
self.lineQueue.put(None)
# AND clear the line event so we don't try to read
# anymore data (and get time-out/prompts) until the
# user has sent some command
self.lineEvent.clear()
def input(self):
# return the next available data line (which may be None for
# time-out/prompt. This will BLOCK if called when the queue
# is empty
return self.lineQueue.get()
def empty(self):
# return the state of the line queue
return self.lineQueue.empty()
def send(self, data, timeout = 1.0):
# send the data to the subprocess
self.process.stdin.write(data)
# reset the timeout value -- this allows the user to specify
# a long timeout for known number crunchers so that one
# doesn't get an errorneous (early) "prompt" indication
self.timeOut = timeout
# signal the line reader that data has been sent, and it
# should start processing return data
self.lineEvent.set()
def shutdown(self):
self.die = True
self.charEvent.set()
self.charThread.join()
self.lineEvent.set()
self.lineThread.join()
myCommand = Shell("cmd.exe")
print "\n**** START of captured output"
while True:
if not myCommand.empty():
# have buffered returns
ln = myCommand.input()
if ln is None: break #prompt found?
print ln
print "\n**** END of captured output"
myCommand.send("ipconfig\n")
print "\n**** START of captured output"
while True:
if not myCommand.empty():
# have buffered returns
ln = myCommand.input()
if ln is None: break #prompt found?
print ln
print "\n**** END of captured output"
myCommand.send("dir\n", timeout=5.0)
print "\n**** START of captured output"
while True:
if not myCommand.empty():
# have buffered returns
ln = myCommand.input()
if ln is None: break #prompt found?
print ln,
print "\n**** END of captured output"
myCommand.send("type script2.py\n")
print "\n**** START of captured output"
while True:
if not myCommand.empty():
# have buffered returns
ln = myCommand.input()
if ln is None: break #prompt found?
print ln,
print "\n**** END of captured output"
myCommand.send("exit\n")
myCommand.shutdown()
-=-=-=-=-=-=-=-=-
d:\data\dbieber\My Documents>python script2.py
**** START of captured output
Microsoft Windows XP [Version 5.1.2600]
(C) Copyright 1985-2001 Microsoft Corp.
d:\data\dbieber\My Documents>
**** END of captured output
**** START of captured output
ipconfig
Windows IP Configuration
Ethernet adapter Local Area Connection:
Connection-specific DNS Suffix . : suppressed.data.com
IP Address. . . . . . . . . . . . : XXX.197.174.84
Subnet Mask . . . . . . . . . . . : 255.255.255.0
Default Gateway . . . . . . . . . : XXX.197.174.254
Ethernet adapter {C8C3DF27-306F-XXXX-AE2A-4F93DAA1E334}:
Connection-specific DNS Suffix . :
IP Address. . . . . . . . . . . . : 0.0.0.0
Subnet Mask . . . . . . . . . . . : 0.0.0.0
Default Gateway . . . . . . . . . :
d:\data\dbieber\My Documents>
**** END of captured output
**** START of captured output
dir
Volume in drive D is Data
Volume Serial Number is 0CBD-XXXX
Directory of d:\data\dbieber\My Documents
10/25/2005 03:51 PM <DIR> .
10/25/2005 03:51 PM <DIR> ..
06/02/2004 10:41 AM <DIR> .idlerc
09/25/2005 03:24 PM 13,711
114674_000E0C316C06C2ADD949120500003699.zip
07/27/2005 04:26 PM 129,798 123160main_cas-skr2-072504.wav
07/27/2005 04:25 PM 734,972 123163main_cas-skr1-112203.wav
09/16/2004 11:15 AM 24,683 2xmodtxt.gz
06/10/2005 04:12 PM 4,517,006 AA.pdf
08/25/2005 04:07 PM <DIR> Ada Progs
08/17/2005 01:49 PM 687,176 ada95_craft_oo_prog.zip
08/17/2005 01:53 PM 1,015,402 adadistilled.pdf
06/13/2005 05:01 PM 750,190 an-introduction-to-tkinter.pdf
06/09/2005 11:17 AM 16,896 art stuff.xls
06/17/2005 03:12 PM 9,490 BookList.log
02/17/2005 12:21 PM 2,174,976 BookList.mdb
10/26/2004 04:32 PM 6,836 BookList.sql
08/05/2005 07:09 PM 8,632 Booklist.zip
11/09/2004 05:18 PM 845 Boromir FTP.lnk
11/03/2004 06:02 PM 798 Boromir Telnet.lnk
07/01/2005 01:02 PM 14,313 c1_1_b.JPG
03/14/2005 12:29 PM <DIR> CDRW Image
08/17/2005 01:56 PM 18,930 chap1.html
08/17/2005 01:56 PM 136,287 chap2.html
08/17/2005 01:56 PM 81,335 chap3.html
08/17/2005 01:57 PM 27,332 chap4.html
10/05/1998 02:27 AM 907,264 COMtut.PPT
08/08/2005 04:09 PM 716,894 COMtut.zip
11/17/2004 04:01 PM 462,848 db1.mdb
07/07/2005 11:18 AM 8,024 Deep Sky3.jpg
10/06/2005 11:38 AM 416 default.gpr
05/29/2003 01:42 PM 25,600 Dell.doc
03/17/2005 12:18 PM 10,985 DH.py
09/07/2005 01:41 PM 52,736 DLB Resume SW Engr locked.doc
09/08/2005 04:00 PM 1,667,306 Draft#3.pdf.pdf
09/08/2005 11:18 AM 497,737 drizzle_update.pdf
01/06/2005 12:09 PM 3,706 DTD.py
07/26/2005 10:35 AM 26,614 eclipse_code.xml
02/24/2005 01:53 PM 3,316 Eco.py
02/24/2005 01:47 PM 6,972 Eco.pyc
01/12/2005 02:57 PM 23,040 Equifax complaint.doc
01/06/2005 12:02 PM 3,356 FCS.py
09/08/2005 03:58 PM 152,691 Focusing%20for%20Beginners.pdf
05/02/2005 04:52 PM 3,232,612 FR_2.6.5_User_Guide.pdf
06/06/2005 11:25 AM <DIR> GKL-II
10/05/2005 06:02 PM <DIR> GSTS_Config
03/15/2005 02:52 PM <DIR> GSTS_OLD
03/03/2005 02:26 PM 862,403 ink_color_charts.pdf
03/16/2005 11:48 AM <DIR> Installation
10/25/2005 03:35 PM <DIR> Installers
08/17/2005 02:22 PM 647,620 intro.html
08/17/2005 02:22 PM <DIR> intro_files
06/21/2005 03:02 PM <DIR> Java
01/07/2003 12:27 PM 4,445,260 kjv10.txt
07/13/2005 03:34 PM 1,403,262 kjv10.zip
08/24/2004 06:02 PM <DIR> Laptop Effort
09/09/2003 10:18 AM 182,407 XXXXlocatormap.pdf
09/25/2005 02:46 PM 49,152 Master Resume.doc
09/25/2005 02:46 PM 63,195 Master Resume.pdf
03/11/2005 04:21 PM 129,368 Mottishaw_Order_Form.pdf
11/22/2004 12:32 PM <DIR> My Downloads
02/11/2005 11:52 AM <DIR> My eBooks
03/15/2005 11:53 AM <DIR> My Music
06/02/2004 10:42 AM <DIR> My Pictures
08/05/2005 02:25 PM 70,201 OMONDO_features_chart.pdf
10/18/2005 05:51 PM 4,813 overload.html
10/05/2005 06:29 PM 63,278
PersonalObjectivesDennisBieber[1].rtf
07/22/2005 07:51 PM <DIR> Python Progs
08/17/2005 01:57 PM 3,300 refs.html
06/10/2005 04:16 PM 2,738,281 RM.pdf
04/27/2005 06:44 PM 56,750
sata_comparison_w_ultra_ata_tech.pdf
10/18/2004 01:02 PM 27,874 SBCPrice.html
09/19/2005 11:09 AM 2,181 scan.py
04/01/2005 04:49 PM 3,415 Script1.py
10/25/2005 03:51 PM 7,145 Script2.py
09/03/2004 03:47 PM 3,884 serial.zip
08/23/2005 03:56 PM 35,328 Sharps Disposal.doc
06/02/2004 10:42 AM <DIR> Spec-Specs
10/15/2005 07:53 PM 7,529 SSCR.log
10/21/2005 06:40 PM <DIR> TCS
07/24/2005 03:06 PM 3,213 TCSreq.csv
07/24/2005 03:06 PM 2,704 tcsreq.csv_c2r.html
07/24/2005 03:08 PM 4,544 tcsreq.csv_r2c.html
06/13/2005 05:05 PM 842,827 tkinter.pdf
10/02/2005 06:28 PM 25,504 tlm-i.svg
08/08/2005 11:08 AM 22,385 untitled1.png
07/03/2003 10:17 AM 32,541 wr-arms.png
06/14/2004 05:27 PM 14,336 Wt_HP chart.xls
12/10/2003 01:30 PM 1,089 Zipper.py
07/20/2005 11:20 AM 308,127 zvprthlp.chm
69 File(s) 30,235,641 bytes
20 Dir(s) 18,418,667,520 bytes free
d:\data\dbieber\My Documents>
**** END of captured output
**** START of captured output
type script2.py
import subprocess
import os
import threading
import Queue
class Shell(object):
def __init__(self, command):
# flag for shutdown
self.die = False
self.timeOut = 1.0
# queue for buffering "lines"
self.lineQueue = Queue.Queue()
# list for collecting characters read from subprocess
# allows for character reader to collect characters
# while line reader processes a prior batch
self.charList = []
# event to signal that, at least, one character is
# available. event is used to allow time-out wait()
self.charEvent = threading.Event()
# event to signal that something has been written to the
# subprocess, and that the line reader should start
# collecting stuff (this is used to avoid multiple "prompt"
# timeouts from falsely affecting the user
self.lineEvent = threading.Event()
# lock to wrap access to self.charList
self.charLock = threading.Lock()
# create subprocess object with supplied command
self.process = subprocess.Popen(command,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE)
# initialize, and start the character reader
# thread -- this is the thread that actually reads the
# subprocess output stream, one character at a time
self.charThread = threading.Thread(target=self.readChar)
self.charThread.start()
# initialize, and start the line reader thread
# -- this is the thread that processes collected characters
# OR on a timeout of the event, queues up a prompt and
# then busy-waits until the queue is empty (meaning the
# user application has processed up to
self.lineThread = threading.Thread(target=self.readLine)
self.lineThread.start()
# set the line event so the line reader knows it should read
up
# to the next time-out/prompt
self.lineEvent.set()
def readChar(self):
while not self.die:
# read one character at a time from subprocess; blocks
ch = self.process.stdout.read(1)
# get the lock to access the character list, append it
self.charLock.acquire()
self.charList.append(ch)
# signal the line reader that, at least one, character
# is in the list
self.charEvent.set()
# let the line reader access the character list
self.charLock.release()
def readLine(self):
lineChar = []
while not self.die:
# block until told that the subprocess has been commanded
# and thereby should be producing some sort of output
self.lineEvent.wait()
# block for up to one second waiting for the character
# reader to produce some sort of output
self.charEvent.wait(self.timeOut)
# get the lock allowing access to the character list
self.charLock.acquire()
# clear the character reader event so the next pass
# will block(time-out) if no more data
self.charEvent.clear()
# save the character list locally so the character reader
# can add more stuff to it (after clearing it)
tCharList = self.charList
self.charList = []
# release the character list lock so reader can touch it
self.charLock.release()
# if the list is not empty, we have some data
if tCharList:
# process each character looking for a new line
character
# note: no filtering is done, all characters will be
# returned to user
for ch in tCharList:
lineChar.append(ch)
# on a new line, collect all preceding and make a
# string out of them, queue the string
if ch == "\n":
self.lineQueue.put("".join(lineChar))
lineChar = []
else:
# time out, assume prompt stage, queue partial line
self.lineQueue.put("".join(lineChar))
lineChar = []
# queue None as signal
self.lineQueue.put(None)
# AND clear the line event so we don't try to read
# anymore data (and get time-out/prompts) until the
# user has sent some command
self.lineEvent.clear()
def input(self):
# return the next available data line (which may be None for
# time-out/prompt. This will BLOCK if called when the queue
# is empty
return self.lineQueue.get()
def empty(self):
# return the state of the line queue
return self.lineQueue.empty()
def send(self, data, timeout = 1.0):
# send the data to the subprocess
self.process.stdin.write(data)
# reset the timeout value -- this allows the user to specify
# a long timeout for known number crunchers so that one
# doesn't get an errorneous (early) "prompt" indication
self.timeOut = timeout
# signal the line reader that data has been sent, and it
# should start processing return data
self.lineEvent.set()
def shutdown(self):
self.die = True
self.charEvent.set()
self.charThread.join()
self.lineEvent.set()
self.lineThread.join()
myCommand = Shell("cmd.exe")
print "\n**** START of captured output"
while True:
if not myCommand.empty():
# have buffered returns
ln = myCommand.input()
if ln is None: break #prompt found?
print ln
print "\n**** END of captured output"
myCommand.send("ipconfig\n")
print "\n**** START of captured output"
while True:
if not myCommand.empty():
# have buffered returns
ln = myCommand.input()
if ln is None: break #prompt found?
print ln
print "\n**** END of captured output"
myCommand.send("dir\n", timeout=5.0)
print "\n**** START of captured output"
while True:
if not myCommand.empty():
# have buffered returns
ln = myCommand.input()
if ln is None: break #prompt found?
print ln,
print "\n**** END of captured output"
myCommand.send("type script2.py\n")
print "\n**** START of captured output"
while True:
if not myCommand.empty():
# have buffered returns
ln = myCommand.input()
if ln is None: break #prompt found?
print ln,
print "\n**** END of captured output"
myCommand.send("exit\n")
myCommand.shutdown()
d:\data\dbieber\My Documents>
**** END of captured output
d:\data\dbieber\My Documents>
--
