I am facing an issue while decoding json string using json.loads

[sajuptpm]

I am facing an issue while decoding json string using json.loads(jstring). Its working, if i do json.dumps(eval(jstring)) before json.loads(jstring). I could not figure out the issue. I want to avoide use of "eval" here.



########## Failing without json.dumps(eval(jstring)) ############


def format_json_string(self, jstring):
"""
"""
from pprint import pprint
print "===string before urllib.unquote===", pprint(jstring)
jstring = urllib.unquote(jstring)##For revert encode change applied in the javascript
print "===string after urllib.unquote===", pprint(jstring)
jstring = json.loads(jstring)
print "===string after json.loads(jstring) ===", pprint(jstring)




===string from client (javascript)========u'{"selected_objects":{"datacenter-2":{"name":"Data !@#$%25^&*()_ {}[]|%5c%2f.,?><:\\"`Center8(Data !@#$%^&*()_ {}[]|\\\\/.,?><:\\"`Center8)","type":"Datacenter","moid":"datacenter-2","actual_name":"Data !@#$%25^&*()_ {}[]|%5c%2f.,?><:\\"`Center8","server_pools":[{"name":"cluster ~!@#$%25^&*()_ {}|\\":?><,.%2f;\'%5c][=-`(cluster ~!@#$%^&*()_ {}|\\":?><,./;\'\\\\][=-`)","type":"ClusterComputeResource","moid":"domain-c401","actual_name":"cluster ~!@#$%25^&*()_ {}|\\":?><,.%2f;\'%5c][=-`","hosts":[{"name":"192.168.1.13","type":"HostSystem","moid":"host-27","actual_name":"192.168.1.13"}]}]}}}'
None



===string after urllib.unquote========u'{"selected_objects":{"datacenter-2":{"name":"Data !@#$%^&*()_ {}[]|\\/.,?><:\\"`Center8(Data !@#$%^&*()_ {}[]|\\\\/.,?><:\\"`Center8)","type":"Datacenter","moid":"datacenter-2","actual_name":"Data !@#$%^&*()_ {}[]|\\/.,?><:\\"`Center8","server_pools":[{"name":"cluster ~!@#$%^&*()_ {}|\\":?><,./;\'\\][=-`(cluster ~!@#$%^&*()_ {}|\\":?><,./;\'\\\\][=-`)","type":"ClusterComputeResource","moid":"domain-c401","actual_name":"cluster ~!@#$%^&*()_ {}|\\":?><,./;\'\\][=-`","hosts":[{"name":"192.168.1.13","type":"HostSystem","moid":"host-27","actual_name":"192.168.1.13"}]}]}}}'
None
Traceback (most recent call last):
File "/home/saju/cvt/tk/src/cct/web/cct/cct/controllers/VcenterController..py", line 69, in import_managed_objects_from_vcenter
self.vcenter_service.validate_vcenter(session['auth'], vcenter_id, context)
File "/home/saju/cvt/tk/src/cct/web/cct/cct/viewModel/VcenterService.py",line 360, in validate_vcenter
context = self.format_json_string(context)
File "/home/saju/cvt/tk/src/cct/web/cct/cct/viewModel/VcenterService.py",line 747, in format_json_string
jstring = json.loads(jstring)
File "/home/saju/cms/cct-enterprise/tg2env/local/lib/python2.7/site-packages/simplejson-2.5.0-py2.7-linux-x86_64.egg/simplejson/__init__.py", line 451, in loads
return _default_decoder.decode(s)
File "/home/saju/cms/cct-enterprise/tg2env/local/lib/python2.7/site-packages/simplejson-2.5.0-py2.7-linux-x86_64.egg/simplejson/decoder.py", line 402, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/home/saju/cms/cct-enterprise/tg2env/local/lib/python2.7/site-packages/simplejson-2.5.0-py2.7-linux-x86_64.egg/simplejson/decoder.py", line 418, in raw_decode
obj, end = self.scan_once(s, idx)
JSONDecodeError: Invalid \escape: line 1 column 289 (char 289)




########### Working with json.dumps(eval(jstring)) #########



But, here i have to do json.dumps(eval(jstring)) before json.loads(). How to do it without eval ???



def format_json_string(self, jstring):
"""
"""
from pprint import pprint
print "===string from client (javascript)===", pprint(jstring)
jstring = urllib.unquote(jstring)##For revert encode change applied in the javascript
print "===string after urllib.unquote===", pprint(jstring)
jstring = json.dumps(eval(jstring))
print "===string after json.dumps(eval(jstring) ===", pprint(jstring)
jstring = json.loads(jstring)
print "===string after json.loads(jstring) ===", pprint(jstring)




===string from client (javascript)========u'{"selected_objects":{"datacenter-2":{"name":"Data !@#$%25^&*()_ {}[]|%5c%2f.,?><:\\"`Center8(Data !@#$%^&*()_ {}[]|\\\\/.,?><:\\"`Center8)","type":"Datacenter","moid":"datacenter-2","actual_name":"Data !@#$%25^&*()_ {}[]|%5c%2f.,?><:\\"`Center8","server_pools":[{"name":"cluster ~!@#$%25^&*()_ {}|\\":?><,.%2f;\'%5c][=-`(cluster ~!@#$%^&*()_ {}|\\":?><,./;\'\\\\][=-`)","type":"ClusterComputeResource","moid":"domain-c401","actual_name":"cluster ~!@#$%25^&*()_ {}|\\":?><,.%2f;\'%5c][=-`","hosts":[{"name":"192.168.1.13","type":"HostSystem","moid":"host-27","actual_name":"192.168.1.13"}]}]}}}'
None



===string after urllib.unquote===u'{"selected_objects":{"datacenter-2":{"name":"Data !@#$%^&*()_ {}[]|\\/.,?><:\\"`Center8(Data !@#$%^&*()_ {}[]|\\\\/.,?><:\\"`Center8)","type":"Datacenter","moid":"datacenter-2","actual_name":"Data !@#$%^&*()_ {}[]|\\/.,?><:\\"`Center8","server_pools":[{"name":"cluster ~!@#$%^&*()_ {}|\\":?><,./;\'\\][=-`(cluster ~!@#$%^&*()_{}|\\":?><,./;\'\\\\][=-`)","type":"ClusterComputeResource","moid":"domain-c401","actual_name":"cluster ~!@#$%^&*()_ {}|\\":?><,./;\'\\][=-`","hosts":[{"name":"192.168.1.13","type":"HostSystem","moid":"host-27","actual_name":"192.168.1.13"}]}]}}}'
None



===string after json.dumps(eval(jstring)========'{"selected_objects": {"datacenter-2": {"actual_name": "Data !@#$%^&*()_ {}[]|\\\\/.,?><:\\"`Center8", "moid": "datacenter-2", "type": "Datacenter", "name": "Data !@#$%^&*()_ {}[]|\\\\/.,?><:\\"`Center8(Data !@#$%^&*()_ {}[]|\\\\/..,?><:\\"`Center8)", "server_pools": [{"hosts": [{"actual_name": "192.168.1..13", "moid": "host-27", "type": "HostSystem", "name": "192.168.1.13"}], "actual_name": "cluster ~!@#$%^&*()_ {}|\\":?><,./;\'\\\\][=-`", "moid": "domain-c401", "type": "ClusterComputeResource", "name": "cluster ~!@#$%^&*()_ {}|\\":?><,./;\'\\\\][=-`(cluster ~!@#$%^&*()_ {}|\\":?><,./;\'\\\\][=-`)"}]}}}'
None



===string after json.loads(jstring)========{'selected_objects': {'datacenter-2': {'actual_name': 'Data !@#$%^&*()_ {}[]|\\/.,?><:"`Center8',
'moid': 'datacenter-2',
'name': 'Data !@#$%^&*()_ {}[]|\\/.,?><:"`Center8(Data !@#$%^&*()_ {}[]|\\/.,?><:"`Center8)',
'server_pools': [{'actual_name': 'cluster ~!@#$%^&*()_ {}|":?><,./;\'\\][=-`',
'hosts': [{'actual_name': '192.168.1.13',
'moid':'host-27',
'name':'192.168.1.13',
'type':'HostSystem'}],
'moid': 'domain-c401',
'name': 'cluster ~!@#$%^&*()_ {}|":?><,./;\'\\][=-`(cluster ~!@#$%^&*()_ {}|":?><,./;\'\\][=-`)',
'type': 'ClusterComputeResource'}],
'type': 'Datacenter'}}}

 

[alex23]

I am facing an issue while decoding json string using json.loads(jstring)..
Its working, if i do json.dumps(eval(jstring)) before json.loads(jstring)..

I cannot reproduce your problem using the original string:

s = u'{"selected_objects":{"datacenter-2":{"name":"Data !@#$%25^&*()_{}[]|%

5c%2f.,?><:\\"`Center8(Data !@#$%^&*()_ {}[]|\\\\/.,?><:\
\"`Center8)","type":"Da
tacenter","moid":"datacenter-2","ac tual_name":"Data !@#$%25^&*()_ {}
[]|%5c%2f.,
?><:\\"`Center8","server_pools":[{"name":"cluster ~!@#$%25^&*()_ {}|\
\":?><,.%2f
;\'%5c][=-`(cluster ~!@#$%^&*()_ {}|\\":?><,./;\'\\\\][=-
`)","type":"ClusterComp
uteResource","moid":"domain- c401","actual_name":"cluster ~!@#$
%25^&*()_ {}|\\":
?><,.%2f;\'%5c][=-`","hosts":[{"name":"192.168.1.13","type":"HostSys
tem","moid"
:"host-27","actual_name":"192.168.1.13"}]}]}}}'{u'selected_objects': {u'datacenter-2': {u'ac tual_name': u'Data !@#$
%25^&*()_ {
}[]|%5c%2f.,?><:"`Center8', u'moid': u'datacenter-2', u'type':
u'Datacenter', u'
name': u'Data !@#$%25^&*()_ {}[]|%5c%2f.,?><:"`Center8(Data !@#$
%^&*()_ {}[]|\\/
..,?><:"`Center8)', u'server_pools': [{u'hosts': [{u'actual_name':
u'192.168.1.13
', u'moid': u'host-27', u'type': u'HostSys tem', u'name':
u'192.168.1.13'}], u'a
ctual_name': u'cluster ~!@#$%25^&*()_ {}|":?><,.%2f;\'%5c][=-`',
u'moid': u'doma
in- c401', u'type': u'ClusterComputeResource', u'name': u'cluster ~!@#$
%25^&*()_

Notice the space in the first occurance of the key 'actual_name' (and
later in 'HostSys tem') however. Could it be a non-printable character
like an EOL that's somehow managed to make its way into your string?

Other than that, I'm at a loss as to why you're seeing a difference.
Which version of Python are you using?

 

[sajuptpm]

Hi,

Fixed:
=====

urllib.unquote is messing up the JSON. Reverse the
order of unquote, and loads, i.e., do a json.loads on
the original string, and then urllib.unquote the components
that need unquote.

Thanks,