🌐 AI搜索 & 代理 主页
Skip to content
This repository was archived by the owner on Sep 8, 2025. It is now read-only.

Commit 147850f

Browse files
cmlenzcmlenz
committed
Add a way to explicitly configure which JSON module should be used, and some fixes for the multipart parsing and dumping.
--HG-- extra : convert_revision : svn%3A7a298fb0-333a-0410-83e7-658617cd9cf3/trunk%40167
1 parent efeff00 commit 147850f

File tree

8 files changed

+83
-64
lines changed

8 files changed

+83
-64
lines changed

‎ChangeLog.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ http://couchdb-python.googlecode.com/svn/tags/0.6.0
2626
(issue 64).
2727
* The `couchdb-dump` tool now operates in a streaming fashion, writing one
2828
document at a time to the resulting MIME multipart file (issue 58).
29+
* It is now possible to explicitly set the JSON module that should be used
30+
for decoding/encoding JSON data. The currently available choices are
31+
`simplejson`, `cjson`, and `json` (the standard library module). It is also
32+
possible to use custom decoding/encoding functions.
2933

3034

3135
Version 0.5

‎couchdb/client.py‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,8 @@
3131
from textwrap import dedent
3232
import re
3333
import socket
34-
try:
35-
import simplejson as json
36-
except ImportError:
37-
import json # Python 2.6
34+
35+
from couchdb import json
3836

3937
__all__ = ['PreconditionFailed', 'ResourceNotFound', 'ResourceConflict',
4038
'ServerError', 'Server', 'Database', 'Document', 'ViewResults',
@@ -701,7 +699,7 @@ def _encode_options(self, options):
701699
for name, value in options.items():
702700
if name in ('key', 'startkey', 'endkey') \
703701
or not isinstance(value, basestring):
704-
value = json.dumps(value, allow_nan=False, ensure_ascii=False)
702+
value = json.encode(value)
705703
retval[name] = value
706704
return retval
707705

@@ -757,8 +755,7 @@ def _exec(self, options):
757755
if 'keys' in options:
758756
options = options.copy()
759757
body['keys'] = options.pop('keys')
760-
content = json.dumps(body, allow_nan=False,
761-
ensure_ascii=False).encode('utf-8')
758+
content = json.encode(body).encode('utf-8')
762759
resp, data = self.resource.post(content=content, headers={
763760
'Content-Type': 'application/json'
764761
}, **self._encode_options(options))
@@ -959,8 +956,7 @@ def _request(self, method, path=None, content=None, headers=None,
959956
body = None
960957
if content is not None:
961958
if not isinstance(content, basestring):
962-
body = json.dumps(content, allow_nan=False,
963-
ensure_ascii=False).encode('utf-8')
959+
body = json.encode(content).encode('utf-8')
964960
headers.setdefault('Content-Type', 'application/json')
965961
else:
966962
body = content
@@ -979,7 +975,7 @@ def _make_request(retry=1):
979975
status_code = int(resp.status)
980976
if data and resp.get('content-type') == 'application/json':
981977
try:
982-
data = json.loads(data)
978+
data = json.decode(data)
983979
except ValueError:
984980
pass
985981

‎couchdb/multipart.py‎

Lines changed: 31 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,19 @@
88

99
"""Support for streamed reading and writing of multipart MIME content."""
1010

11+
from base64 import b64encode
1112
from cgi import parse_header
13+
try:
14+
from hashlib import md5
15+
except ImportError:
16+
from md5 import new as md5
1217
import sys
1318

1419
__all__ = ['read_multipart', 'write_multipart']
1520
__docformat__ = 'restructuredtext en'
1621

1722

18-
EOL = '\r\n'
23+
CRLF = '\r\n'
1924

2025

2126
def read_multipart(fileobj, boundary=None):
@@ -42,18 +47,26 @@ def read_multipart(fileobj, boundary=None):
4247
buf = []
4348
outer = in_headers = boundary is None
4449

45-
next_boundary = boundary and '--' + boundary + EOL or None
46-
last_boundary = boundary and '--' + boundary + '--' + EOL or None
50+
next_boundary = boundary and '--' + boundary + '\n' or None
51+
last_boundary = boundary and '--' + boundary + '--\n' or None
4752

4853
def _current_part():
4954
payload = ''.join(buf)
50-
if payload.endswith(EOL):
55+
if payload.endswith('\r\n'):
56+
payload = payload[:-2]
57+
elif payload.endswith('\n'):
5158
payload = payload[:-1]
59+
content_md5 = headers.get('content-md5')
60+
if content_md5:
61+
h = b64encode(md5(payload).digest())
62+
if content_md5 != h:
63+
raise ValueError('data integrity check failed')
5264
return headers, False, payload
5365

5466
for line in fileobj:
5567
if in_headers:
56-
if line != EOL:
68+
line = line.replace(CRLF, '\n')
69+
if line != '\n':
5770
name, value = line.split(':', 1)
5871
headers[name.lower().strip()] = value.strip()
5972
else:
@@ -71,7 +84,7 @@ def _current_part():
7184
yield part
7285
return
7386

74-
elif line == next_boundary:
87+
elif line.replace(CRLF, '\n') == next_boundary:
7588
# We've reached the start of a new part, as indicated by the
7689
# boundary
7790
if headers:
@@ -83,7 +96,7 @@ def _current_part():
8396
del buf[:]
8497
in_headers = True
8598

86-
elif line == last_boundary:
99+
elif line.replace(CRLF, '\n') == last_boundary:
87100
# We're done with this multipart envelope
88101
break
89102

@@ -111,48 +124,50 @@ def __init__(self, fileobj, headers=None, subtype='mixed', boundary=None):
111124
def open(self, headers=None, subtype='mixed', boundary=None):
112125
self.fileobj.write('--')
113126
self.fileobj.write(self.boundary)
114-
self.fileobj.write(EOL)
127+
self.fileobj.write(CRLF)
115128
return MultipartWriter(self.fileobj, headers=headers, subtype=subtype,
116129
boundary=boundary)
117130

118131
def add(self, mimetype, content, headers=None):
119132
self.fileobj.write('--')
120133
self.fileobj.write(self.boundary)
121-
self.fileobj.write(EOL)
134+
self.fileobj.write(CRLF)
122135
if headers is None:
123136
headers = {}
124137
headers['Content-Type'] = mimetype
125-
headers['Content-Length'] = str(len(content))
138+
if content:
139+
headers['Content-Length'] = str(len(content))
140+
headers['Content-MD5'] = b64encode(md5(content).digest())
126141
self._write_headers(headers)
127142
if content:
128143
# XXX: throw an exception if a boundary appears in the content??
129144
self.fileobj.write(content)
130-
self.fileobj.write(EOL)
145+
self.fileobj.write(CRLF)
131146

132147
def close(self):
133148
self.fileobj.write('--')
134149
self.fileobj.write(self.boundary)
135150
self.fileobj.write('--')
136-
self.fileobj.write(EOL)
151+
self.fileobj.write(CRLF)
137152

138153
def _make_boundary(self):
139154
try:
140155
from uuid import uuid4
141-
return uuid4().hex
156+
return '==' + uuid4().hex + '=='
142157
except ImportError:
143158
from random import randrange
144159
token = randrange(sys.maxint)
145160
format = '%%0%dd' % len(repr(sys.maxint - 1))
146-
return ('=' * 15) + (fmt % token) + '=='
161+
return '===============' + (fmt % token) + '=='
147162

148163
def _write_headers(self, headers):
149164
if headers:
150165
for name in sorted(headers.keys()):
151166
self.fileobj.write(name)
152167
self.fileobj.write(': ')
153168
self.fileobj.write(headers[name])
154-
self.fileobj.write(EOL)
155-
self.fileobj.write(EOL)
169+
self.fileobj.write(CRLF)
170+
self.fileobj.write(CRLF)
156171

157172
def __enter__(self):
158173
return self

‎couchdb/tests/multipart.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
class ReadMultiPartTestCase(unittest.TestCase):
1717

1818
def test_flat(self):
19-
text = '''MIME-Version: 1.0
19+
text = '''\
2020
Content-Type: multipart/mixed; boundary="===============1946781859=="
2121
2222
--===============1946781859==
@@ -59,7 +59,7 @@ def test_flat(self):
5959
self.assertEqual(num, 2)
6060

6161
def test_nested(self):
62-
text = '''MIME-Version: 1.0
62+
text = '''\
6363
Content-Type: multipart/mixed; boundary="===============1946781859=="
6464
6565
--===============1946781859==

‎couchdb/tools/dump.py‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,10 @@
1111
from email.MIMEBase import MIMEBase
1212
from email.MIMEMultipart import MIMEMultipart
1313
from optparse import OptionParser
14-
try:
15-
import simplejson as json
16-
except ImportError:
17-
import json # Python 2.6
1814
import sys
1915

2016
from couchdb import __version__ as VERSION
17+
from couchdb import json
2118
from couchdb.client import Database
2219
from couchdb.multipart import write_multipart
2320

@@ -29,42 +26,43 @@ def dump_db(dburl, username=None, password=None, boundary=None,
2926
db.resource.http.add_credentials(username, password)
3027

3128
envelope = write_multipart(output)
32-
#envelope = MIMEMultipart('mixed', boundary)
3329

3430
for docid in db:
3531
doc = db.get(docid, attachments=True)
3632
print>>sys.stderr, 'Dumping document %r' % doc.id
3733
attachments = doc.pop('_attachments', {})
38-
jsondoc = json.dumps(doc, sort_keys=True, indent=2)
34+
jsondoc = json.encode(doc)
3935

4036
if attachments:
41-
inner = envelope.start({
37+
parts = envelope.open({
4238
'Content-ID': doc.id,
4339
'ETag': '"%s"' % doc.rev
4440
})
45-
part = inner.add('application/json', jsondoc)
41+
parts.add('application/json', jsondoc)
4642

4743
for name, info in attachments.items():
4844
content_type = info.get('content_type')
4945
if content_type is None: # CouchDB < 0.8
5046
content_type = info.get('content-type')
51-
subpart = inner.add(content_type, b64decode(info['data']), {
47+
parts.add(content_type, b64decode(info['data']), {
5248
'Content-ID': name
5349
})
54-
inner.end()
50+
parts.close()
5551

5652
else:
57-
part = envelope.add('application/json', jsondoc, {
53+
envelope.add('application/json', jsondoc, {
5854
'Content-ID': doc.id,
5955
'ETag': '"%s"' % doc.rev
6056
}, )
6157

62-
# XXX Is something supposed to happen here?
63-
#envelope.end()
58+
envelope.close()
6459

6560

6661
def main():
6762
parser = OptionParser(usage='%prog [options] dburl', version=VERSION)
63+
parser.add_option('--json-module', action='store', dest='json_module',
64+
help='the JSON module to use ("simplejson", "cjson", '
65+
'or "json" are supported)')
6866
parser.add_option('-u', '--username', action='store', dest='username',
6967
help='the username to use for authentication')
7068
parser.add_option('-p', '--password', action='store', dest='password',
@@ -75,6 +73,9 @@ def main():
7573
if len(args) != 1:
7674
return parser.error('incorrect number of arguments')
7775

76+
if options.json_module:
77+
json.use(options.json_module)
78+
7879
dump_db(args[0], username=options.username, password=options.password)
7980

8081

‎couchdb/tools/load.py‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,10 @@
1010
from base64 import b64encode
1111
from email import message_from_file
1212
from optparse import OptionParser
13-
try:
14-
import simplejson as json
15-
except ImportError:
16-
import json # Python 2.6
1713
import sys
1814

1915
from couchdb import __version__ as VERSION
16+
from couchdb import json
2017
from couchdb.client import Database
2118
from couchdb.multipart import read_multipart
2219

@@ -32,7 +29,7 @@ def load_db(fileobj, dburl, username=None, password=None, ignore_errors=False):
3229
if is_multipart: # doc has attachments
3330
for headers, _, payload in payload:
3431
if 'content-id' not in headers:
35-
doc = json.loads(payload)
32+
doc = json.decode(payload)
3633
doc['_attachments'] = {}
3734
else:
3835
doc['_attachments'][headers['content-id']] = {
@@ -42,7 +39,7 @@ def load_db(fileobj, dburl, username=None, password=None, ignore_errors=False):
4239
}
4340

4441
else: # no attachments, just the JSON
45-
doc = json.loads(payload)
42+
doc = json.decode(payload)
4643

4744
del doc['_rev']
4845
print>>sys.stderr, 'Loading document %r' % docid
@@ -62,6 +59,9 @@ def main():
6259
dest='ignore_errors',
6360
help='whether to ignore errors in document creation '
6461
'and continue with the remaining documents')
62+
parser.add_option('--json-module', action='store', dest='json_module',
63+
help='the JSON module to use ("simplejson", "cjson", '
64+
'or "json" are supported)')
6565
parser.add_option('-u', '--username', action='store', dest='username',
6666
help='the username to use for authentication')
6767
parser.add_option('-p', '--password', action='store', dest='password',
@@ -73,10 +73,13 @@ def main():
7373
return parser.error('incorrect number of arguments')
7474

7575
if options.input != '-':
76-
fileobj = open(options.input)
76+
fileobj = open(options.input, 'rb')
7777
else:
7878
fileobj = sys.stdin
7979

80+
if options.json_module:
81+
json.use(options.json_module)
82+
8083
load_db(fileobj, args[0], username=options.username,
8184
password=options.password, ignore_errors=options.ignore_errors)
8285

‎couchdb/tools/replication_helper.py‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,9 @@
3333
import os
3434
import sys
3535
import time
36-
try:
37-
import simplejson as json
38-
except ImportError:
39-
import json # Python 2.6
36+
4037
from couchdb import __version__ as VERSION
38+
from couchdb import json
4139

4240

4341
class ReplicationHelper(object):
@@ -67,7 +65,7 @@ def trigger_replication(self, database):
6765
self.http.request(
6866
self.concat_uri(self.args.source_server, '_replicate'),
6967
'POST',
70-
body=json.dumps(body, ensure_ascii=False))
68+
body=json.encode(body))
7169

7270
def trigger_creation(self, database):
7371
"""Creates database in all --target-servers"""
@@ -120,7 +118,7 @@ def __call__(self):
120118
# poor man's validation. If we get garbage, we sys.exit
121119
if not line.endswith('}\n'):
122120
sys.exit(0)
123-
note = json.loads(line)
121+
note = json.decode(line)
124122

125123
# we don't care for deletes
126124
if note['type'] == 'delete' and not args.ignore_deletes:

0 commit comments

Comments
 (0)