string formatting with mapping & '*'... is this a bug?

[Andrew Dalke]

Alex

Ah, sorry, my favourite idiom to avoid the boilerplate of

def __init__(self, fee, fie, foo, fum):
self.fee = fee
self.fie = fie
self.foo = foo
self.fum = fum

A "clever" (a-hem solution can use decorators.
It's hard to make the call signature correct though.
Here's a hack solution using exec, which is the wrong
way to do it. For example, it gets the nested scope
wrong for the context of the original function. But
I'm not Raymond Hettinger and I can't fiddle Python's
byte code well enough for a more correct solution.



# Ugly hack! Do not use this! Only for grins!

def store_args(f):
co = f.func_code
# I think this is right way to get the parameter list
argnames = co.co_varnames[:co.co_argcount]
if argnames[:1] != ("self",):
raise TypeError("first parameter must be 'self'")
argnames = argnames[1:]

argdef = list(argnames)
if f.func_defaults:
start = len(argnames) - len(f.func_defaults)
for i in range(len(f.func_default)):
argdef[start+i] = "%s = _defaults[%s]" % (argdef[start+i], i)

lines = ["def %s(self, %s):" % (f.func_name, ", ".join(argdef))]
for argname in argnames:
lines.append(" self.%s = %s" % (argname, argname))

lines.append(" return __func(self, %s)\n" %
(", ".join(argnames), ))

code = "\n".join(lines)

g = f.func_globals.copy()
g["__func"] = f
d = {"_defaults": f.func_defaults}

exec code in g, d
return d[f.func_name]


class Spam:
@store_args
def __init__(self, x, y, z):
pass


Andrew
(e-mail address removed)

 

[Pierre Fortin]

That's a given in my book... :>

SHEESH!!! Just now dawned on me what you really meant...

# hoist *this* part of the formatting code...!
format = ( "%(Date)ss %(Open)sf %(High)sf %(Low)sf "
"%(Close)sf %(Volume)sd %(AdjClose)sf %(Change)ss"
% fmt )

for ...:
...
print format % map

At least I'm getting it... :>

Pierre
-30-

 

[Bengt Richter]

Well, there is 'once' here -- the names of the attributes are listed
once, in the argument list of 'def __init__'


I don't see how to do that cleanly, simply and within general rules.
"Special cases are not special enough to break the rules"...

You could use a helper function to strip out the name, which you know
anyway if you know to write this.__dict__ or self.__dict__, e.g.,
... sansfirst = locd.copy()
... del sansfirst[firstname]
... locd[firstname].__dict__.update(sansfirst)
... ... def __init__(a1,a2,a3): updd('a1',locals())
... {'a3': 3, 'a2': 2}

The locd.copy etc could be replaced by not using update. Actually, you could
probably inherit from a class with a metaclass that would let you write
just def __init__(a1, a2, a3): pass and have all the methods' __init__s
appropriately modified.

BTW, do you know offhand why somedict.update(obj) seems to use obj.keys() and
obj.__getitem__ if obj has object as base, but not if it has dict as base?
Ran into that trying to wrap locals() to filter out self or whatever:
... def keys(self): return [1,2,3]
... def __getitem__(self, key): return 2*key
...

>>> dd = D({'a':123})
>>> dd.keys() [1, 2, 3]
>>> dd[5] 10
>>> dd {'a': 123}
>>> d={}
>>> d.update(dd)
>>> d

{'a': 123}

Ok, now this: ... def keys(self): return [1,2,3]
... def __getitem__(self, key): return 2*key
... def __init__(self, *a): print a
...

>>> ee = E({'a':123}) ({'a': 123},)
>>> ee.keys() [1, 2, 3]
>>> ee[3] 6
>>> ee

>>> e={}
>>> e.update(ee)
>>> e

{1: 2, 2: 4, 3: 6}

The methods are there and do override the keys and __getitem__ of dict:

>>> dd.keys(), ee.keys() ([1, 2, 3], [1, 2, 3])
>>> dd[5], ee[5]

(10, 10)


But that d.update(dd) doesn't match what you'd think from
---- Help on built-in function update:

update(...)
D.update(E) -> None. Update D from E: for k in E.keys(): D[k] = E[k]
----

I see [<class '__main__.E'>, <type 'object'>]

Is there another method that update looks for first that serves update better
than keys() and __getitem__ so it ignores the latter? Presumably a method that
dict would have but not object?

ISTM I recall a similar problem with some builtin functions not calling overriding
subclass methods as one might expect?

Regards,
Bengt Richter

 

[Alex Martelli]

SHEESH!!! Just now dawned on me what you really meant...

# hoist *this* part of the formatting code...!
format = ( "%(Date)ss %(Open)sf %(High)sf %(Low)sf "
"%(Close)sf %(Volume)sd %(AdjClose)sf %(Change)ss"
% fmt )

for ...:
...
print format % map

At least I'm getting it... :>

Yep, that's exactly what I meant.


Alex

 

[Alex Martelli]

A "clever" (a-hem solution can use decorators.
True.

It's hard to make the call signature correct though.

Maybe module inspect could help -- it's generally the best way to
introspect function signatures in Python. But, perhaps accepting some
sloppiness at the margins, we might code something like (untested!!!):

import itertools

def store_args(f):
co = f.func_code
args = co.co_varnames[1:co.co_argcount]
defs = dict(itertools.izip(reversed(args), f.func_defaults or ()))
def inner(self, *as, **kw):
kkw = dict(defs, **kw)
kkw.update(itertools.izip(args, as))
for kv in kkw.iteritems(): setattr(self, *kv)
return f(self, *as, **kw)
inner.__name__ = f.__name__
return inner

class foo:
@store_args
def __init__(self, a, b, c): pass
def __str__(self): return str(self.__dict__)

f = foo(1, 2, 3); print f


You could add sanity checks such as, f is not allowed a *foo argument
(it makes no sense in this context for f to get arguments of which it
does not know the names). But apart from such issues at the margins,
the signature check happens at inner's call to f -- if any arguments are
missing or not recognized, that call will raise. So, inner's job is
just to consider names and values with the right priority: defaults
first (least priority), then explicitly passed ones (either order since
they can't clash -- if they do the call to f will raise). Here, I build
the auxiliary dict kkw just that way.

I think I still prefer good old self.__dict__.update(locals()),
personally: it's way simpler. Of course, this one has some pluses, such
as calls to setattr rather than __dict__ updates, so it will work in
presence of __slots__, properties, or other weird descriptors, while my
favourite, simpler, more explicit solution doesn't.


Alex

 

[Andrew Dalke]

Me:

Alex:
Maybe module inspect could help -- it's generally the best way to
introspect function signatures in Python. But, perhaps accepting some
sloppiness at the margins, we might code something like (untested!!!):

I had considered that approach. My concern is that
with @decorators current introspection techniques, like
help(), pydoc, and tooltips will become less useful.

For example, here's what help() does on your 'foo' class.

class foo
| Methods defined here:
|
| __init__(self, *as, **kw)
|
| __str__(self)

The inability to propogate the call signature seems to
be an ugly limitation to decorators, and I don't see
an elegant solution to it. I've thought of some hacks,
like calling a function with the original function (f)
and the wrapped function (g), to have it return a new
function (h) with the right signature so that calling
h() pivots and passes things to g() so it can do it's
work then call f().

Matters gets even more complicated with @decorator chains.

BTW, I thought there might be a problem already with
classmethod and staticmethod but it looks like the inspect
module knows how to peer inside to get what it needs.
The same trick won't work for user-defined @decorator
wrappers.

Andrew
(e-mail address removed)

 

[Peter Otten]

I've written a class that automatically generates a format string ready for
dual application of '%'. Instead of
' 1.23'

you can now write
' 1.23'

which is slightly more readable. Not tested beyond what you see.

Peter

import re

class Format:
"""
Extends the format string to allow dict substitution for width and
precision.
' 1.234'
prec=2)
'1.23 '

"""
_cache = {}
# Generously allow all ascii characters as format specifiers
rOuter = re.compile(r"(%(\(.*?\)|[^A-Za-z%])*[A-Za-z%])")
rInner = re.compile(r"\*\(.*?\)")

def __init__(self, format):
self.format = self.prepare(format)

def subInner(self, match):
# called for every width/prec specifier, e. g. "*(width)"
s = match.group(0)
return "%" + s[1:] + "s"

def subOuter(self, match):
# called for every complete format, e. g. "%(value)*(width)s"
s = match.group(0)
if s == "%%":
return "%%%%"
return "%" + self.rInner.sub(self.subInner, s)

def prepare(self, format):
""" Modify the format for a two-pass 'format % dict % dict'
appliction. The first pass replaces width/prec specifiers
with integer literals
"""
cached = self._cache.get(format)
if cached is not None:
return cached
result = self._cache[format] = self.rOuter.sub(self.subOuter,
format)
return result

def __mod__(self, dict):
return self.format % dict % dict

if __name__ == "__main__":
f = Format("%(value)*(width).*(prec)f (literal) "
"%(string)s [%(integer)3d] %% [%(integer)-*(width)d]")
print f % dict(value=1.2345, width=5, prec=2, string="so what",
integer=11)


# Abusing your code as a test case...
fmt = { 'wDate':10, 'wOpen':6, 'wHigh':6, 'wLow':6, # width
'wClose':6, 'wVolume':10, 'wAdjClose':6,
'pDate':10, 'pOpen':2, 'pHigh':2, 'pLow':2, # precision
'pClose':2, 'pVolume':0, 'pAdjClose':2 }

# data will be read from several thousand files
sampledata = [
"9-Sep-04,19.49,20.03,19.35,19.93,60077400,19.93",
"8-Sep-04,18.96,19.53,18.92,18.97,52020600,18.96",
"7-Sep-04,18.98,19.18,18.84,18.85,45498100,18.84",
]

change=["down","up","n/c"]

for D in sampledata:
Date, Open, High, Low, Close, Volume, AdjClose = D.split(',')

map = dict(Date=Date,
Open=float(Open),
High=float(High),
Low=float(Low),
Close=float(Close),
Volume=int(Volume),
AdjClose=float(AdjClose),
#
Change=change[int(float(AdjClose) >= float(Open)) +
int(float(AdjClose) == float(Open))]
)
map.update(fmt)


new = Format(
"%(Date)*(wDate).*(pDate)s "
"%(Open)*(wOpen).*(pOpen)f "
"%(High)*(wHigh).*(pHigh)f "
"%(Low)*(wLow).*(pLow)f "
"%(Close)*(wClose).*(pClose)f "
"%(Volume)*(wVolume).*(pVolume)d "
"%(AdjClose)*(wAdjClose).*(pAdjClose)f "
"%(Change)s") % map

old = (
"%%(Date)%(wDate)d.%(pDate)ds "
"%%(Open)%(wOpen)d.%(pOpen)df "
"%%(High)%(wHigh)d.%(pHigh)df "
"%%(Low)%(wLow)d.%(pLow)df "
"%%(Close)%(wClose)d.%(pClose)df "
"%%(Volume)%(wVolume)d.%(pVolume)dd "
"%%(AdjClose)%(wAdjClose)d.%(pAdjClose)df "
"%%(Change)s") % fmt % map

assert old == new