From 98f3a482ac1dc1d55c559b8636e86fe798f0f046 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 10 Jun 2016 15:43:08 -0600 Subject: [PATCH] Save an entire minion cache traversal on each master pub We were doing an extra and unecessary full traversal of the minion cache. This refactors all returners and the master to reduce this to a single call. More work will be forthcoming to reduce this even further, especially in the context of auth lookups. --- doc/topics/releases/carbon.rst | 7 +++++++ salt/master.py | 2 +- salt/returners/cassandra_cql_return.py | 2 +- salt/returners/couchbase_return.py | 2 +- salt/returners/django_return.py | 2 +- salt/returners/elasticsearch_return.py | 2 +- salt/returners/etcd_return.py | 2 +- salt/returners/influxdb_return.py | 2 +- salt/returners/memcache_return.py | 2 +- salt/returners/mongo_future_return.py | 2 +- salt/returners/multi_returner.py | 2 +- salt/returners/mysql.py | 2 +- salt/returners/odbc.py | 2 +- salt/returners/pgjsonb.py | 2 +- salt/returners/postgres.py | 2 +- salt/returners/postgres_local_cache.py | 2 +- salt/returners/redis_return.py | 2 +- salt/returners/sqlite3_return.py | 2 +- 18 files changed, 24 insertions(+), 17 deletions(-) diff --git a/doc/topics/releases/carbon.rst b/doc/topics/releases/carbon.rst index 7b26e70ca3..6399c723af 100644 --- a/doc/topics/releases/carbon.rst +++ b/doc/topics/releases/carbon.rst @@ -39,6 +39,13 @@ Beacons Changes - The ``loadavg`` beacon now outputs averages as integers instead of strings. (Via :issuse:`31124`.) +Returner Changes +================ + +- Any returner which implements a `save_load` function is now required to + accept a `minions` keyword argument. All returners which ship with Salt + have been modified to do so. + External Module Packaging ========================= diff --git a/salt/master.py b/salt/master.py index eabe60d4c6..56f9f27241 100644 --- a/salt/master.py +++ b/salt/master.py @@ -2266,7 +2266,7 @@ class ClearFuncs(object): if self.opts['ext_job_cache']: try: fstr = '{0}.save_load'.format(self.opts['ext_job_cache']) - self.mminion.returners[fstr](clear_load['jid'], clear_load) + self.mminion.returners[fstr](clear_load['jid'], clear_load, minions=minions) except KeyError: log.critical( 'The specified returner used for the external job cache ' diff --git a/salt/returners/cassandra_cql_return.py b/salt/returners/cassandra_cql_return.py index 4e8768b10c..d0c6be1e96 100644 --- a/salt/returners/cassandra_cql_return.py +++ b/salt/returners/cassandra_cql_return.py @@ -236,7 +236,7 @@ def event_return(events): raise -def save_load(jid, load): +def save_load(jid, load, minions=None): ''' Save the load to the specified jid id ''' diff --git a/salt/returners/couchbase_return.py b/salt/returners/couchbase_return.py index 557f08fd98..74d521979f 100644 --- a/salt/returners/couchbase_return.py +++ b/salt/returners/couchbase_return.py @@ -184,7 +184,7 @@ def returner(load): return False -def save_load(jid, clear_load): +def save_load(jid, clear_load, minion=None): ''' Save the load to the specified jid ''' diff --git a/salt/returners/django_return.py b/salt/returners/django_return.py index 2c1eaf390a..888cf56d8b 100644 --- a/salt/returners/django_return.py +++ b/salt/returners/django_return.py @@ -65,7 +65,7 @@ def returner(ret): 'which responded with {1}'.format(signal[0], signal[1])) -def save_load(jid, load): +def save_load(jid, load, minions=None): ''' Save the load to the specified jid ''' diff --git a/salt/returners/elasticsearch_return.py b/salt/returners/elasticsearch_return.py index 1856b6d64b..33ffdf5a15 100644 --- a/salt/returners/elasticsearch_return.py +++ b/salt/returners/elasticsearch_return.py @@ -164,7 +164,7 @@ def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() -def save_load(jid, load): +def save_load(jid, load, minions=None): ''' Save the load to the specified jid id diff --git a/salt/returners/etcd_return.py b/salt/returners/etcd_return.py index 72e544cfb3..3e0e75f67b 100644 --- a/salt/returners/etcd_return.py +++ b/salt/returners/etcd_return.py @@ -133,7 +133,7 @@ def returner(ret): client.set(dest, json.dumps(ret[field]), ttl=ttl) -def save_load(jid, load): +def save_load(jid, load, minions=None): ''' Save the load to the specified jid ''' diff --git a/salt/returners/influxdb_return.py b/salt/returners/influxdb_return.py index ae0b632bae..d62efa418e 100644 --- a/salt/returners/influxdb_return.py +++ b/salt/returners/influxdb_return.py @@ -189,7 +189,7 @@ def returner(ret): log.critical('Failed to store return with InfluxDB returner: {0}'.format(ex)) -def save_load(jid, load): +def save_load(jid, load, minions=None): ''' Save the load to the specified jid ''' diff --git a/salt/returners/memcache_return.py b/salt/returners/memcache_return.py index a6709be831..ea007c140e 100644 --- a/salt/returners/memcache_return.py +++ b/salt/returners/memcache_return.py @@ -154,7 +154,7 @@ def returner(ret): _append_list(serv, 'jids', jid) -def save_load(jid, load): +def save_load(jid, load, minions=None): ''' Save the load to the specified jid ''' diff --git a/salt/returners/mongo_future_return.py b/salt/returners/mongo_future_return.py index 1f26b1d437..470fa9102e 100644 --- a/salt/returners/mongo_future_return.py +++ b/salt/returners/mongo_future_return.py @@ -199,7 +199,7 @@ def returner(ret): mdb.saltReturns.insert(sdata.copy()) -def save_load(jid, load): +def save_load(jid, load, minions=None): ''' Save the load for a given job id ''' diff --git a/salt/returners/multi_returner.py b/salt/returners/multi_returner.py index da3d22c880..33db562508 100644 --- a/salt/returners/multi_returner.py +++ b/salt/returners/multi_returner.py @@ -61,7 +61,7 @@ def returner(load): _mminion().returners['{0}.returner'.format(returner_)](load) -def save_load(jid, clear_load): +def save_load(jid, clear_load, minions=None): ''' Write load to all returners in multi_returner ''' diff --git a/salt/returners/mysql.py b/salt/returners/mysql.py index 40a5b41876..cd37f124d9 100644 --- a/salt/returners/mysql.py +++ b/salt/returners/mysql.py @@ -287,7 +287,7 @@ def event_return(events): cur.execute(sql, (tag, json.dumps(data), __opts__['id'])) -def save_load(jid, load): +def save_load(jid, load, minions=None): ''' Save the load to the specified jid id ''' diff --git a/salt/returners/odbc.py b/salt/returners/odbc.py index 0319e70399..5dcdd62bdb 100644 --- a/salt/returners/odbc.py +++ b/salt/returners/odbc.py @@ -214,7 +214,7 @@ def returner(ret): _close_conn(conn) -def save_load(jid, load): +def save_load(jid, load, minions=None): ''' Save the load to the specified jid id ''' diff --git a/salt/returners/pgjsonb.py b/salt/returners/pgjsonb.py index 7f5fc6598c..feed2320da 100644 --- a/salt/returners/pgjsonb.py +++ b/salt/returners/pgjsonb.py @@ -272,7 +272,7 @@ def event_return(events): __opts__['id'], time.strftime('%Y-%m-%d %H:%M:%S %z', time.localtime()))) -def save_load(jid, load): +def save_load(jid, load, minions=None): ''' Save the load to the specified jid id ''' diff --git a/salt/returners/postgres.py b/salt/returners/postgres.py index fbb08e301d..aa2a48ac22 100644 --- a/salt/returners/postgres.py +++ b/salt/returners/postgres.py @@ -189,7 +189,7 @@ def returner(ret): _close_conn(conn) -def save_load(jid, load): +def save_load(jid, load, minions=None): ''' Save the load to the specified jid id ''' diff --git a/salt/returners/postgres_local_cache.py b/salt/returners/postgres_local_cache.py index 1c72059c64..583ecfdc9b 100644 --- a/salt/returners/postgres_local_cache.py +++ b/salt/returners/postgres_local_cache.py @@ -264,7 +264,7 @@ def event_return(events): _close_conn(conn) -def save_load(jid, clear_load): +def save_load(jid, clear_load, minions=None): ''' Save the load to the specified jid id ''' diff --git a/salt/returners/redis_return.py b/salt/returners/redis_return.py index e7acc505f5..52a18b7103 100644 --- a/salt/returners/redis_return.py +++ b/salt/returners/redis_return.py @@ -128,7 +128,7 @@ def returner(ret): pipeline.execute() -def save_load(jid, load): +def save_load(jid, load, minions=None): ''' Save the load to the specified jid ''' diff --git a/salt/returners/sqlite3_return.py b/salt/returners/sqlite3_return.py index 2a4d7d51df..44fbbdc68d 100644 --- a/salt/returners/sqlite3_return.py +++ b/salt/returners/sqlite3_return.py @@ -178,7 +178,7 @@ def returner(ret): _close_conn(conn) -def save_load(jid, load): +def save_load(jid, load, minions=None): ''' Save the load to the specified jid '''