From b552c853edfdac834b1220007e601af5f96ac712 Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Fri, 22 Jul 2016 10:37:03 -0600 Subject: [PATCH 1/4] Add load_reg and save_reg for Thorium --- salt/returners/local_cache.py | 44 +++++++++++++++++++++++++++++++++++ salt/thorium/__init__.py | 13 ++++++++++- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/salt/returners/local_cache.py b/salt/returners/local_cache.py index 12fbc80b8a..4f73a1fe04 100644 --- a/salt/returners/local_cache.py +++ b/salt/returners/local_cache.py @@ -478,3 +478,47 @@ def get_endtime(jid): with salt.utils.fopen(etpath, 'r') as etfile: endtime = etfile.read().strip('\n') return endtime + + +def _reg_dir(): + ''' + Return the reg_dir for the given job id + ''' + return os.path.join(__opts__['cachedir'], 'thorium') + + +def save_reg(data): + ''' + Save the register to msgpack files + ''' + reg_dir = _reg_dir() + regfile = os.path.join(reg_dir, 'register') + try: + if not os.path.exists(): + os.makedirs(reg_dir) + except OSError as exc: + if exc.errno == errno.EEXIST: + pass + else: + raise + try: + with salt.utils.flopen(regfile, 'a') as fh_: + msgpack.dump(data, fh_) + fh_.close() + except: + log.error('Could not write to msgpack file {0}'.format(opts['outdir'])) + raise + + +def load_reg(): + ''' + Load the register from msgpack files + ''' + reg_dir = _reg_dir() + regfile = os.path.join(reg_dir, 'register') + try: + with salt.utils.flopen(regfile, 'a') as fh_: + return msgpack.load(fh_) + except: + log.error('Could not write to msgpack file {0}'.format(opts['outdir'])) + raise diff --git a/salt/thorium/__init__.py b/salt/thorium/__init__.py index 5e89107408..2f9587be31 100644 --- a/salt/thorium/__init__.py +++ b/salt/thorium/__init__.py @@ -18,6 +18,7 @@ import traceback # Import Salt libs import salt.state +import salt.loader import salt.payload from salt.exceptions import SaltRenderError @@ -43,7 +44,16 @@ class ThorState(salt.state.HighState): opts['file_client'] = 'local' self.opts = opts salt.state.HighState.__init__(self, self.opts, loader='thorium') - self.state.inject_globals = {'__reg__': {}} + + self.returners = salt.loader.returners(self.opts, {}) + self.reg_ret = self.opts.get('register_returner', 'local_cache') + try: + regdata = self.returners['{0}.load_reg'.format(self.reg_ret)]() + except Exception as exc: + log.error(exc) + regdata = {} + + self.state.inject_globals = {'__reg__': regdata} self.event = salt.utils.event.get_master_event( self.opts, self.opts['sock_dir']) @@ -174,4 +184,5 @@ class ThorState(salt.state.HighState): if (start - r_start) > recompile: cache = self.gather_cache() chunks = self.get_chunks() + self.returners['{0}.save_reg'.format(self.reg_ret)](chunks) r_start = time.time() From dcb8f2b983a8e452f17bce449bdb1922fb93ba6d Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Fri, 22 Jul 2016 11:03:04 -0600 Subject: [PATCH 2/4] Allow register_returner to be Non --- salt/thorium/__init__.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/salt/thorium/__init__.py b/salt/thorium/__init__.py index 2f9587be31..56d3e3f2fc 100644 --- a/salt/thorium/__init__.py +++ b/salt/thorium/__init__.py @@ -46,12 +46,13 @@ class ThorState(salt.state.HighState): salt.state.HighState.__init__(self, self.opts, loader='thorium') self.returners = salt.loader.returners(self.opts, {}) - self.reg_ret = self.opts.get('register_returner', 'local_cache') - try: - regdata = self.returners['{0}.load_reg'.format(self.reg_ret)]() - except Exception as exc: - log.error(exc) - regdata = {} + self.reg_ret = self.opts.get('register_returner', None) + if self.reg_ret is not None: + try: + regdata = self.returners['{0}.load_reg'.format(self.reg_ret)]() + except Exception as exc: + log.error(exc) + regdata = {} self.state.inject_globals = {'__reg__': regdata} self.event = salt.utils.event.get_master_event( @@ -184,5 +185,6 @@ class ThorState(salt.state.HighState): if (start - r_start) > recompile: cache = self.gather_cache() chunks = self.get_chunks() - self.returners['{0}.save_reg'.format(self.reg_ret)](chunks) + if self.reg_ret is not None: + self.returners['{0}.save_reg'.format(self.reg_ret)](chunks) r_start = time.time() From 3de9dbffe5a7bdfcdd5c28d2b791eaefdac124d8 Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Fri, 22 Jul 2016 11:41:31 -0600 Subject: [PATCH 3/4] Fix msgpack functions --- salt/returners/local_cache.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/returners/local_cache.py b/salt/returners/local_cache.py index 4f73a1fe04..cdc946aa9f 100644 --- a/salt/returners/local_cache.py +++ b/salt/returners/local_cache.py @@ -23,6 +23,7 @@ import salt.utils.jid import salt.exceptions # Import 3rd-party libs +import msgpack import salt.ext.six as six @@ -502,7 +503,7 @@ def save_reg(data): else: raise try: - with salt.utils.flopen(regfile, 'a') as fh_: + with salt.utils.fopen(regfile, 'a') as fh_: msgpack.dump(data, fh_) fh_.close() except: @@ -517,7 +518,7 @@ def load_reg(): reg_dir = _reg_dir() regfile = os.path.join(reg_dir, 'register') try: - with salt.utils.flopen(regfile, 'a') as fh_: + with salt.utils.fopen(regfile, 'r') as fh_: return msgpack.load(fh_) except: log.error('Could not write to msgpack file {0}'.format(opts['outdir'])) From 590de80010b1ee0df4d6443d0949adf6259a7e0f Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Fri, 22 Jul 2016 11:42:51 -0600 Subject: [PATCH 4/4] Fix __opts__ --- salt/returners/local_cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/returners/local_cache.py b/salt/returners/local_cache.py index cdc946aa9f..7636890e79 100644 --- a/salt/returners/local_cache.py +++ b/salt/returners/local_cache.py @@ -507,7 +507,7 @@ def save_reg(data): msgpack.dump(data, fh_) fh_.close() except: - log.error('Could not write to msgpack file {0}'.format(opts['outdir'])) + log.error('Could not write to msgpack file {0}'.format(__opts__['outdir'])) raise @@ -521,5 +521,5 @@ def load_reg(): with salt.utils.fopen(regfile, 'r') as fh_: return msgpack.load(fh_) except: - log.error('Could not write to msgpack file {0}'.format(opts['outdir'])) + log.error('Could not write to msgpack file {0}'.format(__opts__['outdir'])) raise