You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
639 lines
22 KiB
639 lines
22 KiB
""" |
|
Multi-part parsing for file uploads. |
|
|
|
Exposes one class, ``MultiPartParser``, which feeds chunks of uploaded data to |
|
file upload handlers for processing. |
|
""" |
|
|
|
import cgi |
|
from django.conf import settings |
|
from django.core.exceptions import SuspiciousOperation |
|
from django.utils.datastructures import MultiValueDict |
|
from django.utils.encoding import force_unicode |
|
from django.utils.text import unescape_entities |
|
from django.core.files.uploadhandler import StopUpload, SkipFile, StopFutureHandlers |
|
|
|
__all__ = ('MultiPartParser', 'MultiPartParserError', 'InputStreamExhausted') |
|
|
|
class MultiPartParserError(Exception): |
|
pass |
|
|
|
class InputStreamExhausted(Exception): |
|
""" |
|
No more reads are allowed from this device. |
|
""" |
|
pass |
|
|
|
RAW = "raw" |
|
FILE = "file" |
|
FIELD = "field" |
|
|
|
class MultiPartParser(object): |
|
""" |
|
A rfc2388 multipart/form-data parser. |
|
|
|
``MultiValueDict.parse()`` reads the input stream in ``chunk_size`` chunks |
|
and returns a tuple of ``(MultiValueDict(POST), MultiValueDict(FILES))``. If |
|
``file_upload_dir`` is defined files will be streamed to temporary files in |
|
that directory. |
|
""" |
|
def __init__(self, META, input_data, upload_handlers, encoding=None): |
|
""" |
|
Initialize the MultiPartParser object. |
|
|
|
:META: |
|
The standard ``META`` dictionary in Django request objects. |
|
:input_data: |
|
The raw post data, as a bytestring. |
|
:upload_handler: |
|
An UploadHandler instance that performs operations on the uploaded |
|
data. |
|
:encoding: |
|
The encoding with which to treat the incoming data. |
|
""" |
|
|
|
# |
|
# Content-Type should containt multipart and the boundary information. |
|
# |
|
|
|
content_type = META.get('HTTP_CONTENT_TYPE', META.get('CONTENT_TYPE', '')) |
|
if not content_type.startswith('multipart/'): |
|
raise MultiPartParserError('Invalid Content-Type: %s' % content_type) |
|
|
|
# Parse the header to get the boundary to split the parts. |
|
ctypes, opts = parse_header(content_type) |
|
boundary = opts.get('boundary') |
|
if not boundary or not cgi.valid_boundary(boundary): |
|
raise MultiPartParserError('Invalid boundary in multipart: %s' % boundary) |
|
|
|
|
|
# |
|
# Content-Length should contain the length of the body we are about |
|
# to receive. |
|
# |
|
try: |
|
content_length = int(META.get('HTTP_CONTENT_LENGTH', META.get('CONTENT_LENGTH',0))) |
|
except (ValueError, TypeError): |
|
# For now set it to 0; we'll try again later on down. |
|
content_length = 0 |
|
|
|
if content_length <= 0: |
|
# This means we shouldn't continue...raise an error. |
|
raise MultiPartParserError("Invalid content length: %r" % content_length) |
|
|
|
self._boundary = boundary |
|
self._input_data = input_data |
|
|
|
# For compatibility with low-level network APIs (with 32-bit integers), |
|
# the chunk size should be < 2^31, but still divisible by 4. |
|
self._chunk_size = min(2**31-4, *[x.chunk_size for x in upload_handlers if x.chunk_size]) |
|
|
|
self._meta = META |
|
self._encoding = encoding or settings.DEFAULT_CHARSET |
|
self._content_length = content_length |
|
self._upload_handlers = upload_handlers |
|
|
|
def parse(self): |
|
""" |
|
Parse the POST data and break it into a FILES MultiValueDict and a POST |
|
MultiValueDict. |
|
|
|
Returns a tuple containing the POST and FILES dictionary, respectively. |
|
""" |
|
# We have to import QueryDict down here to avoid a circular import. |
|
from django.http import QueryDict |
|
|
|
encoding = self._encoding |
|
handlers = self._upload_handlers |
|
|
|
limited_input_data = LimitBytes(self._input_data, self._content_length) |
|
|
|
# See if the handler will want to take care of the parsing. |
|
# This allows overriding everything if somebody wants it. |
|
for handler in handlers: |
|
result = handler.handle_raw_input(limited_input_data, |
|
self._meta, |
|
self._content_length, |
|
self._boundary, |
|
encoding) |
|
if result is not None: |
|
return result[0], result[1] |
|
|
|
# Create the data structures to be used later. |
|
self._post = QueryDict('', mutable=True) |
|
self._files = MultiValueDict() |
|
|
|
# Instantiate the parser and stream: |
|
stream = LazyStream(ChunkIter(limited_input_data, self._chunk_size)) |
|
|
|
# Whether or not to signal a file-completion at the beginning of the loop. |
|
old_field_name = None |
|
counters = [0] * len(handlers) |
|
|
|
try: |
|
for item_type, meta_data, field_stream in Parser(stream, self._boundary): |
|
if old_field_name: |
|
# We run this at the beginning of the next loop |
|
# since we cannot be sure a file is complete until |
|
# we hit the next boundary/part of the multipart content. |
|
self.handle_file_complete(old_field_name, counters) |
|
old_field_name = None |
|
|
|
try: |
|
disposition = meta_data['content-disposition'][1] |
|
field_name = disposition['name'].strip() |
|
except (KeyError, IndexError, AttributeError): |
|
continue |
|
|
|
transfer_encoding = meta_data.get('content-transfer-encoding') |
|
field_name = force_unicode(field_name, encoding, errors='replace') |
|
|
|
if item_type == FIELD: |
|
# This is a post field, we can just set it in the post |
|
if transfer_encoding == 'base64': |
|
raw_data = field_stream.read() |
|
try: |
|
data = str(raw_data).decode('base64') |
|
except: |
|
data = raw_data |
|
else: |
|
data = field_stream.read() |
|
|
|
self._post.appendlist(field_name, |
|
force_unicode(data, encoding, errors='replace')) |
|
elif item_type == FILE: |
|
# This is a file, use the handler... |
|
file_name = disposition.get('filename') |
|
if not file_name: |
|
continue |
|
file_name = force_unicode(file_name, encoding, errors='replace') |
|
file_name = self.IE_sanitize(unescape_entities(file_name)) |
|
|
|
content_type = meta_data.get('content-type', ('',))[0].strip() |
|
try: |
|
charset = meta_data.get('content-type', (0,{}))[1].get('charset', None) |
|
except: |
|
charset = None |
|
|
|
try: |
|
content_length = int(meta_data.get('content-length')[0]) |
|
except (IndexError, TypeError, ValueError): |
|
content_length = None |
|
|
|
counters = [0] * len(handlers) |
|
try: |
|
for handler in handlers: |
|
try: |
|
handler.new_file(field_name, file_name, |
|
content_type, content_length, |
|
charset) |
|
except StopFutureHandlers: |
|
break |
|
|
|
for chunk in field_stream: |
|
if transfer_encoding == 'base64': |
|
# We only special-case base64 transfer encoding |
|
try: |
|
chunk = str(chunk).decode('base64') |
|
except Exception, e: |
|
# Since this is only a chunk, any error is an unfixable error. |
|
raise MultiPartParserError("Could not decode base64 data: %r" % e) |
|
|
|
for i, handler in enumerate(handlers): |
|
chunk_length = len(chunk) |
|
chunk = handler.receive_data_chunk(chunk, |
|
counters[i]) |
|
counters[i] += chunk_length |
|
if chunk is None: |
|
# If the chunk received by the handler is None, then don't continue. |
|
break |
|
|
|
except SkipFile, e: |
|
# Just use up the rest of this file... |
|
exhaust(field_stream) |
|
else: |
|
# Handle file upload completions on next iteration. |
|
old_field_name = field_name |
|
else: |
|
# If this is neither a FIELD or a FILE, just exhaust the stream. |
|
exhaust(stream) |
|
except StopUpload, e: |
|
if not e.connection_reset: |
|
exhaust(limited_input_data) |
|
else: |
|
# Make sure that the request data is all fed |
|
exhaust(limited_input_data) |
|
|
|
# Signal that the upload has completed. |
|
for handler in handlers: |
|
retval = handler.upload_complete() |
|
if retval: |
|
break |
|
|
|
return self._post, self._files |
|
|
|
def handle_file_complete(self, old_field_name, counters): |
|
""" |
|
Handle all the signalling that takes place when a file is complete. |
|
""" |
|
for i, handler in enumerate(self._upload_handlers): |
|
file_obj = handler.file_complete(counters[i]) |
|
if file_obj: |
|
# If it returns a file object, then set the files dict. |
|
self._files.appendlist(force_unicode(old_field_name, |
|
self._encoding, |
|
errors='replace'), |
|
file_obj) |
|
break |
|
|
|
def IE_sanitize(self, filename): |
|
"""Cleanup filename from Internet Explorer full paths.""" |
|
return filename and filename[filename.rfind("\\")+1:].strip() |
|
|
|
class LazyStream(object): |
|
""" |
|
The LazyStream wrapper allows one to get and "unget" bytes from a stream. |
|
|
|
Given a producer object (an iterator that yields bytestrings), the |
|
LazyStream object will support iteration, reading, and keeping a "look-back" |
|
variable in case you need to "unget" some bytes. |
|
""" |
|
def __init__(self, producer, length=None): |
|
""" |
|
Every LazyStream must have a producer when instantiated. |
|
|
|
A producer is an iterable that returns a string each time it |
|
is called. |
|
""" |
|
self._producer = producer |
|
self._empty = False |
|
self._leftover = '' |
|
self.length = length |
|
self.position = 0 |
|
self._remaining = length |
|
self._unget_history = [] |
|
|
|
def tell(self): |
|
return self.position |
|
|
|
def read(self, size=None): |
|
def parts(): |
|
remaining = (size is not None and [size] or [self._remaining])[0] |
|
# do the whole thing in one shot if no limit was provided. |
|
if remaining is None: |
|
yield ''.join(self) |
|
return |
|
|
|
# otherwise do some bookkeeping to return exactly enough |
|
# of the stream and stashing any extra content we get from |
|
# the producer |
|
while remaining != 0: |
|
assert remaining > 0, 'remaining bytes to read should never go negative' |
|
|
|
chunk = self.next() |
|
|
|
emitting = chunk[:remaining] |
|
self.unget(chunk[remaining:]) |
|
remaining -= len(emitting) |
|
yield emitting |
|
|
|
out = ''.join(parts()) |
|
return out |
|
|
|
def next(self): |
|
""" |
|
Used when the exact number of bytes to read is unimportant. |
|
|
|
This procedure just returns whatever is chunk is conveniently returned |
|
from the iterator instead. Useful to avoid unnecessary bookkeeping if |
|
performance is an issue. |
|
""" |
|
if self._leftover: |
|
output = self._leftover |
|
self._leftover = '' |
|
else: |
|
output = self._producer.next() |
|
self._unget_history = [] |
|
self.position += len(output) |
|
return output |
|
|
|
def close(self): |
|
""" |
|
Used to invalidate/disable this lazy stream. |
|
|
|
Replaces the producer with an empty list. Any leftover bytes that have |
|
already been read will still be reported upon read() and/or next(). |
|
""" |
|
self._producer = [] |
|
|
|
def __iter__(self): |
|
return self |
|
|
|
def unget(self, bytes): |
|
""" |
|
Places bytes back onto the front of the lazy stream. |
|
|
|
Future calls to read() will return those bytes first. The |
|
stream position and thus tell() will be rewound. |
|
""" |
|
if not bytes: |
|
return |
|
self._update_unget_history(len(bytes)) |
|
self.position -= len(bytes) |
|
self._leftover = ''.join([bytes, self._leftover]) |
|
|
|
def _update_unget_history(self, num_bytes): |
|
""" |
|
Updates the unget history as a sanity check to see if we've pushed |
|
back the same number of bytes in one chunk. If we keep ungetting the |
|
same number of bytes many times (here, 50), we're mostly likely in an |
|
infinite loop of some sort. This is usually caused by a |
|
maliciously-malformed MIME request. |
|
""" |
|
self._unget_history = [num_bytes] + self._unget_history[:49] |
|
number_equal = len([current_number for current_number in self._unget_history |
|
if current_number == num_bytes]) |
|
|
|
if number_equal > 40: |
|
raise SuspiciousOperation( |
|
"The multipart parser got stuck, which shouldn't happen with" |
|
" normal uploaded files. Check for malicious upload activity;" |
|
" if there is none, report this to the Django developers." |
|
) |
|
|
|
class ChunkIter(object): |
|
""" |
|
An iterable that will yield chunks of data. Given a file-like object as the |
|
constructor, this object will yield chunks of read operations from that |
|
object. |
|
""" |
|
def __init__(self, flo, chunk_size=64 * 1024): |
|
self.flo = flo |
|
self.chunk_size = chunk_size |
|
|
|
def next(self): |
|
try: |
|
data = self.flo.read(self.chunk_size) |
|
except InputStreamExhausted: |
|
raise StopIteration() |
|
if data: |
|
return data |
|
else: |
|
raise StopIteration() |
|
|
|
def __iter__(self): |
|
return self |
|
|
|
class LimitBytes(object): |
|
""" Limit bytes for a file object. """ |
|
def __init__(self, fileobject, length): |
|
self._file = fileobject |
|
self.remaining = length |
|
|
|
def read(self, num_bytes=None): |
|
""" |
|
Read data from the underlying file. |
|
If you ask for too much or there isn't anything left, |
|
this will raise an InputStreamExhausted error. |
|
""" |
|
if self.remaining <= 0: |
|
raise InputStreamExhausted() |
|
if num_bytes is None: |
|
num_bytes = self.remaining |
|
else: |
|
num_bytes = min(num_bytes, self.remaining) |
|
self.remaining -= num_bytes |
|
return self._file.read(num_bytes) |
|
|
|
class InterBoundaryIter(object): |
|
""" |
|
A Producer that will iterate over boundaries. |
|
""" |
|
def __init__(self, stream, boundary): |
|
self._stream = stream |
|
self._boundary = boundary |
|
|
|
def __iter__(self): |
|
return self |
|
|
|
def next(self): |
|
try: |
|
return LazyStream(BoundaryIter(self._stream, self._boundary)) |
|
except InputStreamExhausted: |
|
raise StopIteration() |
|
|
|
class BoundaryIter(object): |
|
""" |
|
A Producer that is sensitive to boundaries. |
|
|
|
Will happily yield bytes until a boundary is found. Will yield the bytes |
|
before the boundary, throw away the boundary bytes themselves, and push the |
|
post-boundary bytes back on the stream. |
|
|
|
The future calls to .next() after locating the boundary will raise a |
|
StopIteration exception. |
|
""" |
|
|
|
def __init__(self, stream, boundary): |
|
self._stream = stream |
|
self._boundary = boundary |
|
self._done = False |
|
# rollback an additional six bytes because the format is like |
|
# this: CRLF<boundary>[--CRLF] |
|
self._rollback = len(boundary) + 6 |
|
|
|
# Try to use mx fast string search if available. Otherwise |
|
# use Python find. Wrap the latter for consistency. |
|
unused_char = self._stream.read(1) |
|
if not unused_char: |
|
raise InputStreamExhausted() |
|
self._stream.unget(unused_char) |
|
try: |
|
from mx.TextTools import FS |
|
self._fs = FS(boundary).find |
|
except ImportError: |
|
self._fs = lambda data: data.find(boundary) |
|
|
|
def __iter__(self): |
|
return self |
|
|
|
def next(self): |
|
if self._done: |
|
raise StopIteration() |
|
|
|
stream = self._stream |
|
rollback = self._rollback |
|
|
|
bytes_read = 0 |
|
chunks = [] |
|
for bytes in stream: |
|
bytes_read += len(bytes) |
|
chunks.append(bytes) |
|
if bytes_read > rollback: |
|
break |
|
if not bytes: |
|
break |
|
else: |
|
self._done = True |
|
|
|
if not chunks: |
|
raise StopIteration() |
|
|
|
chunk = ''.join(chunks) |
|
boundary = self._find_boundary(chunk, len(chunk) < self._rollback) |
|
|
|
if boundary: |
|
end, next = boundary |
|
stream.unget(chunk[next:]) |
|
self._done = True |
|
return chunk[:end] |
|
else: |
|
# make sure we dont treat a partial boundary (and |
|
# its separators) as data |
|
if not chunk[:-rollback]:# and len(chunk) >= (len(self._boundary) + 6): |
|
# There's nothing left, we should just return and mark as done. |
|
self._done = True |
|
return chunk |
|
else: |
|
stream.unget(chunk[-rollback:]) |
|
return chunk[:-rollback] |
|
|
|
def _find_boundary(self, data, eof = False): |
|
""" |
|
Finds a multipart boundary in data. |
|
|
|
Should no boundry exist in the data None is returned instead. Otherwise |
|
a tuple containing the indices of the following are returned: |
|
|
|
* the end of current encapsulation |
|
* the start of the next encapsulation |
|
""" |
|
index = self._fs(data) |
|
if index < 0: |
|
return None |
|
else: |
|
end = index |
|
next = index + len(self._boundary) |
|
# backup over CRLF |
|
if data[max(0,end-1)] == '\n': |
|
end -= 1 |
|
if data[max(0,end-1)] == '\r': |
|
end -= 1 |
|
return end, next |
|
|
|
def exhaust(stream_or_iterable): |
|
""" |
|
Completely exhausts an iterator or stream. |
|
|
|
Raise a MultiPartParserError if the argument is not a stream or an iterable. |
|
""" |
|
iterator = None |
|
try: |
|
iterator = iter(stream_or_iterable) |
|
except TypeError: |
|
iterator = ChunkIter(stream_or_iterable, 16384) |
|
|
|
if iterator is None: |
|
raise MultiPartParserError('multipartparser.exhaust() was passed a non-iterable or stream parameter') |
|
|
|
for __ in iterator: |
|
pass |
|
|
|
def parse_boundary_stream(stream, max_header_size): |
|
""" |
|
Parses one and exactly one stream that encapsulates a boundary. |
|
""" |
|
# Stream at beginning of header, look for end of header |
|
# and parse it if found. The header must fit within one |
|
# chunk. |
|
chunk = stream.read(max_header_size) |
|
|
|
# 'find' returns the top of these four bytes, so we'll |
|
# need to munch them later to prevent them from polluting |
|
# the payload. |
|
header_end = chunk.find('\r\n\r\n') |
|
|
|
def _parse_header(line): |
|
main_value_pair, params = parse_header(line) |
|
try: |
|
name, value = main_value_pair.split(':', 1) |
|
except: |
|
raise ValueError("Invalid header: %r" % line) |
|
return name, (value, params) |
|
|
|
if header_end == -1: |
|
# we find no header, so we just mark this fact and pass on |
|
# the stream verbatim |
|
stream.unget(chunk) |
|
return (RAW, {}, stream) |
|
|
|
header = chunk[:header_end] |
|
|
|
# here we place any excess chunk back onto the stream, as |
|
# well as throwing away the CRLFCRLF bytes from above. |
|
stream.unget(chunk[header_end + 4:]) |
|
|
|
TYPE = RAW |
|
outdict = {} |
|
|
|
# Eliminate blank lines |
|
for line in header.split('\r\n'): |
|
# This terminology ("main value" and "dictionary of |
|
# parameters") is from the Python docs. |
|
try: |
|
name, (value, params) = _parse_header(line) |
|
except: |
|
continue |
|
|
|
if name == 'content-disposition': |
|
TYPE = FIELD |
|
if params.get('filename'): |
|
TYPE = FILE |
|
|
|
outdict[name] = value, params |
|
|
|
if TYPE == RAW: |
|
stream.unget(chunk) |
|
|
|
return (TYPE, outdict, stream) |
|
|
|
class Parser(object): |
|
def __init__(self, stream, boundary): |
|
self._stream = stream |
|
self._separator = '--' + boundary |
|
|
|
def __iter__(self): |
|
boundarystream = InterBoundaryIter(self._stream, self._separator) |
|
for sub_stream in boundarystream: |
|
# Iterate over each part |
|
yield parse_boundary_stream(sub_stream, 1024) |
|
|
|
def parse_header(line): |
|
""" Parse the header into a key-value. """ |
|
plist = _parse_header_params(';' + line) |
|
key = plist.pop(0).lower() |
|
pdict = {} |
|
for p in plist: |
|
i = p.find('=') |
|
if i >= 0: |
|
name = p[:i].strip().lower() |
|
value = p[i+1:].strip() |
|
if len(value) >= 2 and value[0] == value[-1] == '"': |
|
value = value[1:-1] |
|
value = value.replace('\\\\', '\\').replace('\\"', '"') |
|
pdict[name] = value |
|
return key, pdict |
|
|
|
def _parse_header_params(s): |
|
plist = [] |
|
while s[:1] == ';': |
|
s = s[1:] |
|
end = s.find(';') |
|
while end > 0 and s.count('"', 0, end) % 2: |
|
end = s.find(';', end + 1) |
|
if end < 0: |
|
end = len(s) |
|
f = s[:end] |
|
plist.append(f.strip()) |
|
s = s[end:] |
|
return plist
|
|
|