diff --git a/salt/fileclient.py b/salt/fileclient.py index ce201a4de8..fdb23311c8 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -593,35 +593,38 @@ class Client(object): get_kwargs['auth'] = (url_data.username, url_data.password) else: fixed_url = url + + destfp = None try: + if no_cache: + result = [] + + def on_chunk(chunk): + result.append(chunk) + else: + dest_tmp = "{0}.part".format(dest) + destfp = salt.utils.fopen(dest_tmp, 'wb') + + def on_chunk(chunk): + destfp.write(chunk) + query = salt.utils.http.query( fixed_url, stream=True, + streaming_callback=on_chunk, username=url_data.username, password=url_data.password, **get_kwargs ) if 'handle' not in query: raise MinionError('Error: {0}'.format(query['error'])) - response = query['handle'] - chunk_size = 32 * 1024 - if not no_cache: - with salt.utils.fopen(dest, 'wb') as destfp: - if hasattr(response, 'iter_content'): - for chunk in response.iter_content(chunk_size=chunk_size): - destfp.write(chunk) - else: - while True: - chunk = response.buffer.read(chunk_size) - destfp.write(chunk) - if len(chunk) < chunk_size: - break - return dest + if no_cache: + return ''.join(result) else: - if hasattr(response, 'text'): - return response.text - else: - return response['text'] + destfp.close() + destfp = None + os.rename(dest_tmp, dest) + return dest except HTTPError as exc: raise MinionError('HTTP error {0} reading {1}: {3}'.format( exc.code, @@ -629,6 +632,9 @@ class Client(object): *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code])) except URLError as exc: raise MinionError('Error reading {0}: {1}'.format(url, exc.reason)) + finally: + if destfp is not None: + destfp.close() def get_template( self, diff --git a/salt/utils/http.py b/salt/utils/http.py index 73c78d0ed2..836948903a 100644 --- a/salt/utils/http.py +++ b/salt/utils/http.py @@ -118,6 +118,7 @@ def query(url, headers_out=None, decode_out=None, stream=False, + streaming_callback=None, handle=False, agent=USERAGENT, hide_fields=None, @@ -403,7 +404,7 @@ def query(url, data = urllib.urlencode(data) try: - result = HTTPClient().fetch( + result = HTTPClient(max_body_size=100*1024*1024*1024).fetch( url_full, method=method, headers=header_dict, @@ -412,6 +413,8 @@ def query(url, body=data, validate_cert=verify_ssl, allow_nonstandard_methods=True, + streaming_callback=streaming_callback, + request_timeout=3600.0, **req_kwargs ) except tornado.httpclient.HTTPError as exc: