valitydev/salt
14
0
Fork 0
mirror of https://github.com/valitydev/salt.git synced 2024-11-07 17:09:03 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #35499 from cro/returner_self_destruct

Browse Source 
Returner self destruct
This commit is contained in:
Mike Place 2016-08-20 11:42:03 +09:00 committed by GitHub
commit e7405258d4
4 changed files with 261 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
salt/config/__init__.py
View File

@ -514,6 +514,10 @@ VALID_OPTS = {
# The number of hours to keep jobs around in the job cache on the master
'keep_jobs': int,
# If the returner supports `clean_old_jobs`, then at cleanup time,
# archive the job data before deleting it.
'archive_jobs': bool,
# A master-only copy of the file_roots dictionary, used by the state compiler
'master_roots': dict,
@ -1161,6 +1165,7 @@ DEFAULT_MASTER_OPTS = {
'ret_port': 4506,
'timeout': 5,
'keep_jobs': 24,
'archive_jobs': False,
'root_dir': salt.syspaths.ROOT_DIR,
'pki_dir': os.path.join(salt.syspaths.CONFIG_DIR, 'pki', 'master'),
'key_cache': '',

134
salt/returners/mysql.py
View File

@ -3,7 +3,7 @@
Return data to a mysql server
:maintainer: Dave Boucha <dave@saltstack.com>, Seth House <shouse@saltstack.com>
:maturity: new
:maturity: mature
:depends: python-mysqldb
:platform: all
@ -44,6 +44,20 @@ optional. The following ssl options are simply for illustration purposes:
alternative.mysql.ssl_cert: '/etc/pki/mysql/certs/localhost.crt'
alternative.mysql.ssl_key: '/etc/pki/mysql/certs/localhost.key'
Should you wish the returner data to be cleaned out every so often, set
`keep_jobs` to the number of hours for the jobs to live in the tables.
Setting it to `0` or leaving it unset will cause the data to stay in the tables.
Should you wish to archive jobs in a different table for later processing,
set `archive_jobs` to True. Salt will create 3 archive tables
- `jids_archive`
- `salt_returns_archive`
- `salt_events_archive`
and move the contents of `jids`, `salt_returns`, and `salt_events` that are
more than `keep_jobs` hours old to these tables.
Use the following mysql database schema:
.. code-block:: sql
@ -439,3 +453,121 @@ def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()
def _purge_jobs(timestamp):
'''
Purge records from the returner tables.
:param job_age_in_seconds: Purge jobs older than this
:return:
'''
with _get_serv() as cur:
try:
sql = 'delete from `jids` where jid in (select distinct jid from salt_returns where alter_time < %s)'
cur.execute(sql, (timestamp,))
cur.execute('COMMIT')
except MySQLdb.Error as e:
log.error('mysql returner archiver was unable to delete contents of table \'jids\'')
log.error(str(e))
raise salt.exceptions.Salt(str(e))
try:
sql = 'delete from `salt_returns` where alter_time < %s'
cur.execute(sql, (timestamp,))
cur.execute('COMMIT')
except MySQLdb.Error as e:
log.error('mysql returner archiver was unable to delete contents of table \'salt_returns\'')
log.error(str(e))
raise salt.exceptions.Salt(str(e))
try:
sql = 'delete from `salt_events` where alter_time < %s'
cur.execute(sql, (timestamp,))
cur.execute('COMMIT')
except MySQLdb.Error as e:
log.error('mysql returner archiver was unable to delete contents of table \'salt_events\'')
log.error(str(e))
raise salt.exceptions.Salt(str(e))
return True
def _archive_jobs(timestamp):
'''
Copy rows to a set of backup tables, then purge rows.
:param timestamp: Archive rows older than this timestamp
:return:
'''
source_tables = ['jids',
'salt_returns',
'salt_events']
with _get_serv() as cur:
target_tables = {}
for table_name in source_tables:
try:
tmp_table_name = table_name + '_archive'
sql = 'create table if not exists {0} like {1}'.format(tmp_table_name, table_name)
cur.execute(sql)
cur.execute('COMMIT')
target_tables[table_name] = tmp_table_name
except MySQLdb.Error as e:
log.error('mysql returner archiver was unable to create the archive tables.')
log.error(str(e))
raise salt.exceptions.SaltRunnerError(str(e))
try:
sql = 'insert into `{0}` select * from `{1}` where jid in (select distinct jid from salt_returns where alter_time < %s)'.format(target_tables['jids'], 'jids')
cur.execute(sql, (timestamp,))
cur.execute('COMMIT')
except MySQLdb.Error as e:
log.error('mysql returner archiver was unable to copy contents of table \'jids\'')
log.error(str(e))
raise salt.exceptions.SaltRunnerError(str(e))
except Exception as e:
log.error(e)
raise
try:
sql = 'insert into `{0}` select * from `{1}` where alter_time < %s'.format(target_tables['salt_returns'], 'salt_returns')
cur.execute(sql, (timestamp,))
cur.execute('COMMIT')
except MySQLdb.Error as e:
log.error('mysql returner archiver was unable to copy contents of table \'salt_returns\'')
log.error(str(e))
raise salt.exceptions.SaltRunnerError(str(e))
try:
sql = 'insert into `{0}` select * from `{1}` where alter_time < %s'.format(target_tables['salt_events'], 'salt_events')
cur.execute(sql, (timestamp,))
cur.execute('COMMIT')
except MySQLdb.Error as e:
log.error('mysql returner archiver was unable to copy contents of table \'salt_events\'')
log.error(str(e))
raise salt.exceptions.SaltRunnerError(str(e))
return _purge_jobs(timestamp)
def clean_old_jobs():
'''
Called in the master's event loop every loop_interval. Archives and/or
deletes the events and job details from the database.
:return:
'''
if __opts__.get('keep_jobs', False) and int(__opts__.get('keep_jobs', 0)) > 0:
try:
with _get_serv() as cur:
sql = 'select now() - %s as stamp;'
cur.execute(sql, (__opts__['keep_jobs'],))
rows = cur.fetchall()
stamp = int(rows[0][0]) * 60 * 60
if __opts__.get('archive_jobs', False):
_archive_jobs(stamp)
else:
_purge_jobs(stamp)
except MySQLdb.Error as e:
log.error('Mysql returner was unable to get timestamp for purge/archive of jobs')
log.error(str(e))
raise salt.exceptions.Salt(str(e))

7
tests/integration/files/conf/master
View File

@ -79,3 +79,10 @@ nodegroups:
redundant_minions: N@min or N@mins
nodegroup_loop_a: N@nodegroup_loop_b
nodegroup_loop_b: N@nodegroup_loop_a
mysql.host: localhost
mysql.user: 'salt'
mysql.pass: 'salt'
mysql.db: 'salt'
mysql.port: 3306

116
tests/integration/files/mysql_returner_archiver_data.sql Normal file
View File

@ -0,0 +1,116 @@
-- MySQL dump 10.15 Distrib 10.0.22-MariaDB, for Linux (x86_64)
--
-- Host: localhost Database: salt
-- ------------------------------------------------------
-- Server version 10.0.22-MariaDB
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
--
-- Table structure for table `jids`
--
CREATE DATABASE if not exists `salt`
DEFAULT CHARACTER SET utf8
DEFAULT COLLATE utf8_general_ci;
USE `salt`;
DROP TABLE IF EXISTS `jids`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `jids` (
`jid` varchar(255) NOT NULL,
`load` mediumtext NOT NULL,
UNIQUE KEY `jid` (`jid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Dumping data for table `jids`
--
LOCK TABLES `jids` WRITE;
/*!40000 ALTER TABLE `jids` DISABLE KEYS */;
INSERT INTO `jids` VALUES ('20160719134843873492','{\"tgt_type\": \"compound\", \"jid\": \"20160719134843873492\", \"tgt\": \"G@virtual:physical and G@os:smartos\", \"cmd\": \"publish\", \"ret\": \"\", \"user\": \"root\", \"arg\": [], \"fun\": \"test.ping\"}'),('20160719134848959936','{\"tgt_type\": \"compound\", \"jid\": \"20160719134848959936\", \"tgt\": \"G@virtual:physical and G@os:smartos\", \"cmd\": \"publish\", \"ret\": \"\", \"user\": \"root\", \"arg\": [\"20160719134843873492\"], \"fun\": \"saltutil.find_job\"}'),('20160719134910163074','{\"tgt_type\": \"glob\", \"jid\": \"20160719134910163074\", \"cmd\": \"publish\", \"tgt\": \"twd\", \"kwargs\": {\"delimiter\": \":\", \"show_timeout\": true, \"show_jid\": false}, \"ret\": \"\", \"user\": \"root\", \"arg\": [], \"fun\": \"test.ping\"}'),('20160719134919147347','{\"tgt_type\": \"glob\", \"jid\": \"20160719134919147347\", \"cmd\": \"publish\", \"tgt\": \"twd\", \"kwargs\": {\"delimiter\": \":\", \"show_timeout\": true, \"show_jid\": false}, \"ret\": \"\", \"user\": \"root\", \"arg\": [], \"fun\": \"network.interfaces\"}'),('20160719135029732667','{\"tgt_type\": \"glob\", \"jid\": \"20160719135029732667\", \"cmd\": \"publish\", \"tgt\": \"twd\", \"kwargs\": {\"delimiter\": \":\", \"show_timeout\": true, \"show_jid\": false}, \"ret\": \"\", \"user\": \"root\", \"arg\": [{\"refresh\": true, \"__kwarg__\": true}], \"fun\": \"pkg.upgrade\"}'),('20160719135034878238','{\"tgt_type\": \"glob\", \"jid\": \"20160719135034878238\", \"cmd\": \"publish\", \"tgt\": \"twd\", \"kwargs\": {\"delimiter\": \":\"}, \"ret\": \"\", \"user\": \"root\", \"arg\": [\"20160719135029732667\"], \"fun\": \"saltutil.find_job\"}'),('20160719135044921491','{\"tgt_type\": \"glob\", \"jid\": \"20160719135044921491\", \"cmd\": \"publish\", \"tgt\": \"twd\", \"kwargs\": {\"delimiter\": \":\"}, \"ret\": \"\", \"user\": \"root\", \"arg\": [\"20160719135029732667\"], \"fun\": \"saltutil.find_job\"}');
/*!40000 ALTER TABLE `jids` ENABLE KEYS */;
UNLOCK TABLES;
--
-- Table structure for table `salt_events`
--
DROP TABLE IF EXISTS `salt_events`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `salt_events` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`tag` varchar(255) NOT NULL,
`data` mediumtext NOT NULL,
`alter_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
`master_id` varchar(255) NOT NULL,
PRIMARY KEY (`id`),
KEY `tag` (`tag`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Dumping data for table `salt_events`
--
LOCK TABLES `salt_events` WRITE;
/*!40000 ALTER TABLE `salt_events` DISABLE KEYS */;
/*!40000 ALTER TABLE `salt_events` ENABLE KEYS */;
UNLOCK TABLES;
--
-- Table structure for table `salt_returns`
--
DROP TABLE IF EXISTS `salt_returns`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `salt_returns` (
`fun` varchar(50) NOT NULL,
`jid` varchar(255) NOT NULL,
`return` mediumtext NOT NULL,
`id` varchar(255) NOT NULL,
`success` varchar(10) NOT NULL,
`full_ret` mediumtext NOT NULL,
`alter_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
KEY `id` (`id`),
KEY `jid` (`jid`),
KEY `fun` (`fun`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Dumping data for table `salt_returns`
--
LOCK TABLES `salt_returns` WRITE;
/*!40000 ALTER TABLE `salt_returns` DISABLE KEYS */;
INSERT INTO `salt_returns` VALUES ('test.ping','20160719134910163074','true','twd','1','{\"fun_args\": [], \"jid\": \"20160719134910163074\", \"return\": true, \"retcode\": 0, \"success\": true, \"cmd\": \"_return\", \"_stamp\": \"2016-07-19T19:49:10.295047\", \"fun\": \"test.ping\", \"id\": \"twd\"}','2016-07-19 19:49:10'),('network.interfaces','20160719134919147347','{\"lo\": {\"hwaddr\": \"00:00:00:00:00:00\", \"up\": true, \"inet\": [{\"broadcast\": null, \"netmask\": \"255.0.0.0\", \"address\": \"127.0.0.1\", \"label\": \"lo\"}], \"inet6\": [{\"prefixlen\": \"128\", \"scope\": \"host\", \"address\": \"::1\"}]}, \"docker0\": {\"hwaddr\": \"02:42:bb:e2:f6:7e\", \"up\": true, \"inet\": [{\"broadcast\": null, \"netmask\": \"255.255.0.0\", \"address\": \"172.17.0.1\", \"label\": \"docker0\"}]}, \"eno16777984\": {\"hwaddr\": \"00:0c:29:e3:6b:c8\", \"up\": true}, \"br0\": {\"hwaddr\": \"00:0c:29:e3:6b:c8\", \"up\": true, \"inet\": [{\"broadcast\": \"172.16.207.255\", \"netmask\": \"255.255.255.0\", \"address\": \"172.16.207.136\", \"label\": \"br0\"}], \"inet6\": [{\"prefixlen\": \"64\", \"scope\": \"link\", \"address\": \"fe80::20c:29ff:fee3:6bc8\"}]}}','twd','1','{\"fun_args\": [], \"jid\": \"20160719134919147347\", \"return\": {\"lo\": {\"hwaddr\": \"00:00:00:00:00:00\", \"up\": true, \"inet\": [{\"broadcast\": null, \"netmask\": \"255.0.0.0\", \"address\": \"127.0.0.1\", \"label\": \"lo\"}], \"inet6\": [{\"prefixlen\": \"128\", \"scope\": \"host\", \"address\": \"::1\"}]}, \"docker0\": {\"hwaddr\": \"02:42:bb:e2:f6:7e\", \"up\": true, \"inet\": [{\"broadcast\": null, \"netmask\": \"255.255.0.0\", \"address\": \"172.17.0.1\", \"label\": \"docker0\"}]}, \"eno16777984\": {\"hwaddr\": \"00:0c:29:e3:6b:c8\", \"up\": true}, \"br0\": {\"hwaddr\": \"00:0c:29:e3:6b:c8\", \"up\": true, \"inet\": [{\"broadcast\": \"172.16.207.255\", \"netmask\": \"255.255.255.0\", \"address\": \"172.16.207.136\", \"label\": \"br0\"}], \"inet6\": [{\"prefixlen\": \"64\", \"scope\": \"link\", \"address\": \"fe80::20c:29ff:fee3:6bc8\"}]}}, \"retcode\": 0, \"success\": true, \"cmd\": \"_return\", \"_stamp\": \"2016-07-19T19:49:19.222588\", \"fun\": \"network.interfaces\", \"id\": \"twd\"}','2016-07-19 19:49:19'),('saltutil.find_job','20160719135034878238','{\"tgt_type\": \"glob\", \"jid\": \"20160719135029732667\", \"tgt\": \"twd\", \"pid\": 5557, \"ret\": \"\", \"user\": \"root\", \"arg\": [{\"refresh\": true, \"__kwarg__\": true}], \"fun\": \"pkg.upgrade\"}','twd','1','{\"fun_args\": [\"20160719135029732667\"], \"jid\": \"20160719135034878238\", \"return\": {\"tgt_type\": \"glob\", \"jid\": \"20160719135029732667\", \"tgt\": \"twd\", \"pid\": 5557, \"ret\": \"\", \"user\": \"root\", \"arg\": [{\"refresh\": true, \"__kwarg__\": true}], \"fun\": \"pkg.upgrade\"}, \"retcode\": 0, \"success\": true, \"cmd\": \"_return\", \"_stamp\": \"2016-07-19T19:50:34.967377\", \"fun\": \"saltutil.find_job\", \"id\": \"twd\"}','2016-07-19 19:50:34'),('saltutil.find_job','20160719135044921491','{\"tgt_type\": \"glob\", \"jid\": \"20160719135029732667\", \"tgt\": \"twd\", \"pid\": 5557, \"ret\": \"\", \"user\": \"root\", \"arg\": [{\"refresh\": true, \"__kwarg__\": true}], \"fun\": \"pkg.upgrade\"}','twd','1','{\"fun_args\": [\"20160719135029732667\"], \"jid\": \"20160719135044921491\", \"return\": {\"tgt_type\": \"glob\", \"jid\": \"20160719135029732667\", \"tgt\": \"twd\", \"pid\": 5557, \"ret\": \"\", \"user\": \"root\", \"arg\": [{\"refresh\": true, \"__kwarg__\": true}], \"fun\": \"pkg.upgrade\"}, \"retcode\": 0, \"success\": true, \"cmd\": \"_return\", \"_stamp\": \"2016-07-19T19:50:45.034813\", \"fun\": \"saltutil.find_job\", \"id\": \"twd\"}','2016-07-19 19:50:45'),('pkg.upgrade','20160719135029732667','{\"comment\": \"\", \"changes\": {}, \"result\": true}','twd','1','{\"fun_args\": [{\"refresh\": true}], \"jid\": \"20160719135029732667\", \"return\": {\"comment\": \"\", \"changes\": {}, \"result\": true}, \"retcode\": 0, \"success\": true, \"cmd\": \"_return\", \"_stamp\": \"2016-07-19T19:50:52.016142\", \"fun\": \"pkg.upgrade\", \"id\": \"twd\"}','2016-07-19 19:50:52');
/*!40000 ALTER TABLE `salt_returns` ENABLE KEYS */;
UNLOCK TABLES;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-- Dump completed on 2016-07-19 13:52:18