valitydev/salt
14
0
Fork 0
mirror of https://github.com/valitydev/salt.git synced 2024-11-07 17:09:03 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add gzip_compression support to cp.get_file

Browse Source
This commit is contained in:
Ryan Schneider 2012-11-08 16:09:38 -08:00
parent 0e70da3588
commit c638617612
8 changed files with 373 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
doc/ref/file_server/index.rst
View File

@ -48,6 +48,20 @@ like so:
This example would instruct all Salt minions to download the vimrc from a
directory with the same name as their os grain and copy it to /etc/vimrc
For larger files, the cp.get_file module also supports gzip compression.
Because gzip compression is CPU-intensive, this should only be used in
scenarios where the compression ratio is very high (e.g. pretty-printed JSON
or YAML files).
Use the *gzip_compression* named argument to enable it. Valid values are 1..9,
where 1 is the lightest compression and 9 the heaviest. 1 uses the least CPU
on the master (and minion), 9 uses the most.
.. code-block:: bash
# salt '*' cp.get_file salt://vimrc /etc/vimrc gzip_compression=5
File Server Client API
----------------------

19
salt/fileclient.py
View File

@ -22,6 +22,8 @@ import salt.utils
import salt.payload
import salt.utils
import salt.utils.templates
import salt.utils.gzip_util
from salt._compat import (
URLError, HTTPError, BaseHTTPServer, urlparse, url_open)
@ -411,9 +413,10 @@ class LocalClient(Client):
return fnd
return fnd
def get_file(self, path, dest='', makedirs=False, env='base'):
def get_file(self, path, dest='', makedirs=False, env='base', gzip_compression=None):
'''
Copies a file from the local files directory into :param:`dest`
gzip_compression settings are ignored for local files
'''
path = self._check_proto(path)
fnd = self._find_file(path, env)
@ -552,7 +555,7 @@ class RemoteClient(Client):
self.auth = salt.crypt.SAuth(opts)
self.sreq = salt.payload.SREQ(self.opts['master_uri'])
def get_file(self, path, dest='', makedirs=False, env='base'):
def get_file(self, path, dest='', makedirs=False, env='base', gzip_compression=None):
'''
Get a single file from the salt-master
path must be a salt server location, aka, salt://path/to/file, if
@ -564,6 +567,10 @@ class RemoteClient(Client):
load = {'path': path,
'env': env,
'cmd': '_serve_file'}
if gzip_compression:
gzip_compression = int(gzip_compression)
load['gzip_compression'] = gzip_compression
fn_ = None
if dest:
destdir = os.path.dirname(dest)
@ -588,6 +595,7 @@ class RemoteClient(Client):
)
except SaltReqTimeoutError:
return ''
if not data['data']:
if not fn_ and data['dest']:
# This is a 0 byte file on the master
@ -601,7 +609,12 @@ class RemoteClient(Client):
with self._cache_loc(data['dest'], env) as cache_dest:
dest = cache_dest
fn_ = open(dest, 'wb+')
fn_.write(data['data'])
gzip_compression = data.get('gzip_compression', None)
if gzip_compression:
data = salt.utils.gzip_util.uncompress(data['data'])
else:
data = data['data']
fn_.write(data)
if fn_:
fn_.close()
return dest

9
salt/master.py
View File

@ -45,6 +45,7 @@ import salt.utils.atomicfile
import salt.utils.event
import salt.utils.verify
import salt.utils.minions
import salt.utils.gzip_util
from salt.utils.debug import enable_sigusr1_handler
@ -644,9 +645,15 @@ class AESFuncs(object):
if not fnd['path']:
return ret
ret['dest'] = fnd['rel']
gzip_compression = load.get('gzip_compression', None)
with open(fnd['path'], 'rb') as fp_:
fp_.seek(load['loc'])
ret['data'] = fp_.read(self.opts['file_buffer_size'])
data = fp_.read(self.opts['file_buffer_size'])
if gzip_compression and data:
data = salt.utils.gzip_util.compress(data, gzip_compression)
ret['gzip_compression'] = gzip_compression
ret['data'] = data
return ret
def _file_hash(self, load):

4
salt/modules/cp.py
View File

@ -42,7 +42,7 @@ def recv(files, dest):
return ret
def get_file(path, dest, env='base', template=None):
def get_file(path, dest, env='base', template=None, gzip_compression=None):
'''
Used to get a single file from the salt master
@ -100,7 +100,7 @@ def get_file(path, dest, env='base', template=None):
return ''
else:
client = salt.fileclient.get_file_client(__opts__)
return client.get_file(path, dest, False, env)
return client.get_file(path, dest, False, env, gzip_compression)
def get_template(path, dest, template='jinja', env='base', **kwargs):

55
salt/utils/gzip_util.py Normal file
View File

@ -0,0 +1,55 @@
# -*- coding: utf-8 -*-
'''
salt.utils.gzip
~~~~~~~~~~~~~~~~~~~~~
Helper module for handling gzip consistently between 2.7+ and 2.6-
'''
import gzip
import StringIO
class GzipFile(gzip.GzipFile):
def __init__(self, filename=None, mode=None,
compresslevel=9, fileobj=None):
gzip.GzipFile.__init__(self, filename, mode, compresslevel, fileobj)
### Context manager (stolen from Python 2.7)###
def __enter__(self):
"""Context management protocol. Returns self."""
return self
def __exit__(self, *args):
"""Context management protocol. Calls close()"""
self.close()
def open(filename, mode="rb", compresslevel=9):
if hasattr(gzip.GzipFile, '__enter__'):
return gzip.open(filename, mode, compresslevel)
else:
return GzipFile(filename, mode, compresslevel)
def open_fileobj(fileobj, mode='rb', compresslevel=9):
if hasattr(gzip.GzipFile, '__enter__'):
return gzip.GzipFile(filename='', mode=mode, fileobj=fileobj, compresslevel=compresslevel)
else:
return GzipFile(filename='', mode=mode, fileobj=fileobj, compresslevel=compresslevel)
def compress(data, compresslevel=9):
'''
Returns the data compressed at gzip level compression.
'''
buffer = StringIO.StringIO()
with open_fileobj(buffer, 'wb', compresslevel) as gz:
gz.write(data)
compressed = buffer.getvalue()
return compressed
def uncompress(data):
buffer = StringIO.StringIO(data)
with open_fileobj(buffer, 'rb') as gz:
unc = gz.read()
return unc

2
tests/integration/files/conf/master
View File

@ -16,3 +16,5 @@ log_file: master
key_logfile: key
pub_refresh: False
token_file: /tmp/ksfjhdgiuebfgnkefvsikhfjdgvkjahcsidk
file_buffer_size: 8192

253
tests/integration/files/file/base/file.big Normal file
View File

@ -0,0 +1,253 @@
The Knights Who Say Ni demand a sacrifice!
Oh, ow! Now, look here, my good man. Ni! Ni! Ni! Ni!
Sets The Cinema Back 900 Years!
Well, Mercia's a temperate zone! I am your king. Burn her! And the hat. She's a witch!
It's only a model.
Oh, ow!
What a strange person
You don't frighten us, English pig-dogs! Go and boil your bottoms, sons of a silly person! I blow my nose at you, so-called Ah-thoor Keeng, you and all your silly English K-n-n-n-n-n-n-n-niggits! The nose? Shut up! Knights of Ni, we are but simple travelers who seek the enchanter who lives beyond these woods.
I'm not dead!
What a strange person. Where'd you get the coconuts? Ni! Ni! Ni! Ni!
The swallow may fly south with the sun, and the house martin or the plover may seek warmer climes in winter, yet these are not strangers to our land.
Did you dress her up like this?
It's only a model.
Where'd you get the coconuts?
King Arthur
Burn her anyway! Where'd you get the coconuts? I'm not a witch.
What… is your quest?
It's only a model. You don't vote for kings. Well, how'd you become king, then? Shut up! Will you shut up?! She looks like one.
I am your king. Well, I didn't vote for you. Now, look here, my good man. We found them. We want a shrubbery!!
But you are dressed as one… The swallow may fly south with the sun, and the house martin or the plover may seek warmer climes in winter, yet these are not strangers to our land. I dunno. Must be a king. What do you mean? Ni! Ni! Ni! Ni!
I am your king. How do you know she is a witch? Camelot!
No, no, no! Yes, yes. A bit. But she's got a wart. The nose? Ni! Ni! Ni! Ni! Well, Mercia's a temperate zone! Oh, ow! Well, how'd you become king, then?
Who's that then? Bring her forward! Found them? In Mercia?! The coconut's tropical! I am your king. Oh! Come and see the violence inherent in the system! Help, help, I'm being repressed!
You don't frighten us, English pig-dogs! Go and boil your bottoms, sons of a silly person! I blow my nose at you, so-called Ah-thoor Keeng, you and all your silly English K-n-n-n-n-n-n-n-niggits! Where'd you get the coconuts? And this isn't my nose. This is a false one. You can't expect to wield supreme power just 'cause some watery tart threw a sword at you! Well, Mercia's a temperate zone! Oh, ow!
We found them. And the hat. She's a witch! We shall say 'Ni' again to you, if you do not appease us. Shut up! Will you shut up?! Shut up!
He hasn't got shit all over him. Why do you think that she is a witch? Well, Mercia's a temperate zone! It's only a model. I'm not a witch. Burn her anyway!
And the hat. She's a witch! Who's that then? I'm not a witch.
No, no, no! Yes, yes. A bit. But she's got a wart. A newt? I dunno. Must be a king. Burn her anyway! But you are dressed as one… The Knights Who Say Ni demand a sacrifice!
…Are you suggesting that coconuts migrate? Bring her forward! Oh, ow!
No, no, no! Yes, yes. A bit. But she's got a wart. Camelot! The swallow may fly south with the sun, and the house martin or the plover may seek warmer climes in winter, yet these are not strangers to our land. And the hat. She's a witch! We want a shrubbery!! …Are you suggesting that coconuts migrate?
How do you know she is a witch? Look, my liege! I'm not a witch.
Why do you think that she is a witch? The Knights Who Say Ni demand a sacrifice! Oh, ow! Where'd you get the coconuts?
We want a shrubbery!!
You can't expect to wield supreme power just 'cause some watery tart threw a sword at you! Now, look here, my good man. Oh, ow! He hasn't got shit all over him. It's only a model. Now, look here, my good man.
Sir Lancelot
But you are dressed as one… Well, Mercia's a temperate zone! Oh, ow! Well, I didn't vote for you.
But you are dressed as one…
Oh, ow!
The nose?
Sets The Cinema Back 900 Years!
We want a shrubbery!! And this isn't my nose. This is a false one. You don't vote for kings. He hasn't got shit all over him. The Knights Who Say Ni demand a sacrifice!
First shalt thou take out the Holy Pin
Ni! Ni! Ni! Ni! Oh, ow! And the hat. She's a witch! You don't frighten us, English pig-dogs! Go and boil your bottoms, sons of a silly person! I blow my nose at you, so-called Ah-thoor Keeng, you and all your silly English K-n-n-n-n-n-n-n-niggits!
Oh, ow!
Well, how'd you become king, then?
We want a shrubbery!!
Where'd you get the coconuts? I am your king. Did you dress her up like this? Found them? In Mercia?! The coconut's tropical! I'm not a witch.
What… is your quest?
I'm not a witch. You don't frighten us, English pig-dogs! Go and boil your bottoms, sons of a silly person! I blow my nose at you, so-called Ah-thoor Keeng, you and all your silly English K-n-n-n-n-n-n-n-niggits! Shut up! Will you shut up?!
Burn her anyway! And the hat. She's a witch! A newt? Knights of Ni, we are but simple travelers who seek the enchanter who lives beyond these woods. The nose?
Where'd you get the coconuts? Camelot! But you are dressed as one…
The swallow may fly south with the sun, and the house martin or the plover may seek warmer climes in winter, yet these are not strangers to our land. And the hat. She's a witch! …Are you suggesting that coconuts migrate?
The swallow may fly south with the sun, and the house martin or the plover may seek warmer climes in winter, yet these are not strangers to our land. Who's that then? How do you know she is a witch? Why do you think that she is a witch?
She looks like one. The nose? Bring her forward! The Knights Who Say Ni demand a sacrifice! Oh, ow! Burn her anyway!
Ni! Ni! Ni! Ni! Ni! Ni! Ni! Ni! The nose? We found them. Well, Mercia's a temperate zone! But you are dressed as one…
Look, my liege! I dunno. Must be a king. Shut up!
We found them. And the hat. She's a witch! Where'd you get the coconuts? I'm not a witch. And the hat. She's a witch! No, no, no! Yes, yes. A bit. But she's got a wart.
We shall say 'Ni' again to you, if you do not appease us. …Are you suggesting that coconuts migrate? I dunno. Must be a king.
Shut up! How do you know she is a witch? Bring her forward!
Bring her forward! She looks like one. I am your king. Camelot! No, no, no! Yes, yes. A bit. But she's got a wart.
Well, how'd you become king, then? Camelot! I'm not a witch. Well, how'd you become king, then? What a strange person.
Burn her! I am your king. Well, I got better. Shut up! Will you shut up?! Where'd you get the coconuts?
It's only a model. Oh! Come and see the violence inherent in the system! Help, help, I'm being repressed! We want a shrubbery!!
Am I right?
Well, how'd you become king, then? He hasn't got shit all over him. Knights of Ni, we are but simple travelers who seek the enchanter who lives beyond these woods. I dunno. Must be a king.
Bridgekeeper
Burn her anyway! Where'd you get the coconuts? Shut up! Will you shut up?! Now, look here, my good man.
And the hat. She's a witch!
Where'd you get the coconuts?
But you are dressed as one…
Help, help, I'm being repressed!
The nose? What do you mean? But you are dressed as one… Who's that then?
What a strange person
But you are dressed as one… Why do you think that she is a witch? Ni! Ni! Ni! Ni! Look, my liege!
I dunno. Must be a king.
Shut up! Will you shut up?!
Shut up!
Burn her anyway!
We want a shrubbery!!
Where'd you get the coconuts? …Are you suggesting that coconuts migrate? I am your king. Oh, ow!
We want a shrubbery!!
Ni! Ni! Ni! Ni! I'm not a witch. She looks like one. Camelot! Oh! Come and see the violence inherent in the system! Help, help, I'm being repressed! She looks like one.
Well, Mercia's a temperate zone! Camelot! I am your king. Shut up! Well, Mercia's a temperate zone! Camelot!
We want a shrubbery!! She looks like one. You don't vote for kings. The Knights Who Say Ni demand a sacrifice! Bring her forward!
No, no, no! Yes, yes. A bit. But she's got a wart. Oh, ow! Camelot! Burn her! Bring her forward!
We shall say 'Ni' again to you, if you do not appease us. Did you dress her up like this? The Knights Who Say Ni demand a sacrifice! Well, how'd you become king, then? We found them. Why do you think that she is a witch?
No, no, no! Yes, yes. A bit. But she's got a wart. And the hat. She's a witch! I'm not a witch.
And the hat. She's a witch! I dunno. Must be a king. I'm not a witch. It's only a model.
And the hat. She's a witch! You don't frighten us, English pig-dogs! Go and boil your bottoms, sons of a silly person! I blow my nose at you, so-called Ah-thoor Keeng, you and all your silly English K-n-n-n-n-n-n-n-niggits! What a strange person.
And this isn't my nose. This is a false one. …Are you suggesting that coconuts migrate? We found them. How do you know she is a witch? And the hat. She's a witch! No, no, no! Yes, yes. A bit. But she's got a wart.
Who's that then? Where'd you get the coconuts? Oh, ow!
Well, Mercia's a temperate zone! A newt? Burn her anyway! We want a shrubbery!! But you are dressed as one… Well, I got better.
Shut up! The nose? Well, Mercia's a temperate zone! Ni! Ni! Ni! Ni! The swallow may fly south with the sun, and the house martin or the plover may seek warmer climes in winter, yet these are not strangers to our land. Bring her forward!
I'm not a witch. Ni! Ni! Ni! Ni! Found them? In Mercia?! The coconut's tropical! Where'd you get the coconuts? He hasn't got shit all over him.
Oh, ow! Shut up! Will you shut up?! Well, I didn't vote for you.
The swallow may fly south with the sun, and the house martin or the plover may seek warmer climes in winter, yet these are not strangers to our land. Burn her anyway! It's only a model.
--------
Scene 33
[clop clop whinny]
KNIGHT: They're nervous, sire.
ARTHUR: Then we'd best leave them here and carry on on foot. Dis-mount!
TIM: Behold the cave of Kyre Banorg!
ARTHUR: Right! Keep me covered.
KNIGHT: What with?
ARTHUR: Just keep me covered.
TIM: Too late!
[chord]
ARTHUR: What?
TIM: There he is!
ARTHUR: Where?
TIM: There!
ARTHUR: What, behind the rabbit?
TIM: It is the {{ spam }}!
ARTHUR: You silly sod! You got us all worked up!
TIM: Well, that's no ordinary rabbit. That's the most foul, cruel,
and bad-tempered rodent you ever set eyes on.
ROBIN: You tit! I soiled my armor I was so scared!
TIM: Look, that rabbit's got a vicious streak a mile wide, it's a
killer!
KNIGHT: Get stuffed!
TIM: It'll do you a trick, mate!
KNIGHT: Oh, yeah?
ROBIN: You mangy Scot git!
TIM: I'm warning you!
ROBIN: What's he do, nibble your bum?
TIM: He's got huge, sharp-- he can leap about-- look at the bones!
ARTHUR: Go on, Boris. Chop his head off!
BORIS: Right! Silly little bleeder. One rabbit stew comin' right up!
TIM: Look!
[squeak]
BORIS: Aaaugh!
[chord]
ARTHUR: Jesus Christ!
TIM: I warned you!
ROBIN: I peed again!
TIM: I warned you! But did you listen to me? Oh, no, you knew it all,
didn't you? Oh, it's just a harmless little bunny, isn't it? Well,
it's always the same, I always--
ARTHUR: Oh, shut up!
TIM: --But do they listen to me?--
ARTHUR: Right!
TIM: -Oh, no--
KNIGHTS: Charge!
[squeak squeak]
KNIGHTS: Aaaaugh! Aaaugh! etc.
KNIGHTS: Run away! Run away!
TIM: Haw haw haw. Haw haw haw. Haw haw.
ARTHUR: Right. How many did we lose?
KNIGHT: Gawain.
KNIGHT: Hector.
ARTHUR: And Boris. That's five.
GALAHAD: Three, sir.
ARTHUR: Three. Three. And we'd better not risk another frontal
assault, that rabbit's dynamite.
ROBIN: Would it help to confuse it if we run away more?
ARTHUR: Oh, shut up and go and change your armor.
GALAHAD: Let us taunt it! It may become so cross that it will make
a mistake.
ARTHUR: Like what?
GALAHAD: Well,....
ARTHUR: Have we got bows?
KNIGHT: No.
LAUNCELOT: We have the Holy Hand Grenade.
ARTHUR: Yes, of course! The Holy Hand Grenade of Antioch! 'Tis one
of the sacred relics Brother Maynard carries with him! Brother Maynard!
Bring up the Holy Hand Grenade!
[singing]
How does it, uh... how does it work?
KNIGHT: I know not, my liege.
ARTHUR: Consult the Book of Armaments!
MAYNARD: Armaments, Chapter Two, Verses Nine to Twenty-One.
BROTHER: "And Saint Atila raised the hand grenade up on high, saying,
'Oh, Lord, bless this thy hand grenade that with it thou mayest blow
thy enemies to tiny bits, in thy mercy.' And the Lord did grin, and
people did feast upon the lambs, and sloths, and carp, and anchovies,
and orangutans, and breakfast cereals, and fruit bats, and large --"
MAYNARD: Skip a bit, Brother.
BROTHER: "And the Lord spake, saying, 'First shalt thou take out the
Holy Pin. Then, shalt thou count to three, no more, no less. Three
shalt be the number thou shalt count, and the number of the counting
shalt be three. Four shalt thou not count, nor either count thou two,
excepting that thou then proceed to three. Five is right out. Once
the number three, being the third number, be reached, then lobbest thou
thy Holy Hand Grenade of Antioch towards thou foe, who being naughty
in my sight, shall snuff it.'"
MAYNARD: Amen.
ALL: Amen.
ARTHUR: Right! One... two... five!
KNIGHT: Three, sir!
ARTHUR: Three!
[boom]

23
tests/integration/modules/cp.py
View File

@ -42,6 +42,29 @@ class CPModuleTest(integration.ModuleCase):
self.assertIn('Gromit', data)
self.assertNotIn('bacon', data)
def test_get_file_gzipped(self):
'''
cp.get_file
'''
tgt = os.path.join(integration.TMP, 'file.big')
src = os.path.join(integration.FILES, 'file/base/file.big')
with open(src, 'r') as fp_:
hash = hashlib.md5(fp_.read()).hexdigest()
self.run_function(
'cp.get_file',
[
'salt://file.big',
tgt,
],
gzip_compression=5
)
with open(tgt, 'r') as scene:
data = scene.read()
self.assertIn('KNIGHT: They\'re nervous, sire.', data)
self.assertNotIn('bacon', data)
self.assertEqual(hash, hashlib.md5(data).hexdigest())
def test_get_template(self):