valitydev/salt
14
0
Fork 0
mirror of https://github.com/valitydev/salt.git synced 2024-11-07 08:58:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Refactor jid utils out of utils/__init__

Browse Source
This commit is contained in:
Mike Place 2014-12-11 15:33:01 -07:00
parent 1f39a65665
commit 5daf851995
27 changed files with 148 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
salt/daemons/masterapi.py
View File

@ -38,6 +38,7 @@ import salt.utils.event
import salt.utils.verify
import salt.utils.minions
import salt.utils.gzip_util
import salt.utils.jid
from salt.pillar import git_pillar
from salt.utils.event import tagify
from salt.exceptions import SaltMasterError
@ -1135,7 +1136,7 @@ class LocalFuncs(object):
return dict(error=dict(name='TokenAuthenticationError',
message=msg))
jid = salt.utils.gen_jid()
jid = salt.utils.jid.gen_jid()
fun = load.pop('fun')
tag = tagify(jid, prefix='wheel')
data = {'fun': "wheel.{0}".format(fun),
@ -1205,7 +1206,7 @@ class LocalFuncs(object):
return dict(error=dict(name='EauthAuthenticationError',
message=msg))
jid = salt.utils.gen_jid()
jid = salt.utils.jid.gen_jid()
fun = load.pop('fun')
tag = tagify(jid, prefix='wheel')
data = {'fun': "wheel.{0}".format(fun),

5
salt/master.py
View File

@ -46,6 +46,7 @@ import salt.utils.minions
import salt.utils.gzip_util
import salt.utils.process
import salt.utils.zeromq
import salt.utils.jid
from salt.defaults import DEFAULT_TARGET_DELIM
from salt.utils.debug import enable_sigusr1_handler, enable_sigusr2_handler, inspect_stack
from salt.utils.event import tagify
@ -2017,7 +2018,7 @@ class ClearFuncs(object):
else:
token = self.loadauth.get_tok(clear_load['token'])
jid = salt.utils.gen_jid()
jid = salt.utils.jid.gen_jid()
fun = clear_load.pop('fun')
tag = tagify(jid, prefix='wheel')
data = {'fun': "wheel.{0}".format(fun),
@ -2049,7 +2050,7 @@ class ClearFuncs(object):
eauth_error = self.process_eauth(clear_load, 'wheel')
if eauth_error:
return eauth_error
jid = salt.utils.gen_jid()
jid = salt.utils.jid.gen_jid()
fun = clear_load.pop('fun')
tag = tagify(jid, prefix='wheel')
data = {'fun': "wheel.{0}".format(fun),

5
salt/minion.py
View File

@ -63,6 +63,7 @@ import salt.crypt
import salt.loader
import salt.payload
import salt.utils
import salt.utils.jid
import salt.utils.args
import salt.utils.event
import salt.utils.minion
@ -2107,7 +2108,7 @@ class Syndic(Minion):
self.event_forward_timeout = (
time.time() + self.opts['syndic_event_forward_timeout']
)
if salt.utils.is_jid(event['tag']) and 'return' in event['data']:
if salt.utils.jid.is_jid(event['tag']) and 'return' in event['data']:
if 'jid' not in event['data']:
# Not a job return
continue
@ -2344,7 +2345,7 @@ class MultiSyndic(MinionBase):
self.event_forward_timeout = (
time.time() + self.opts['syndic_event_forward_timeout']
)
if salt.utils.is_jid(event['tag']) and 'return' in event['data']:
if salt.utils.jid.is_jid(event['tag']) and 'return' in event['data']:
if 'jid' not in event['data']:
# Not a job return
continue

3
salt/modules/state.py
View File

@ -18,6 +18,7 @@ import tempfile
# Import salt libs
import salt.config
import salt.utils
import salt.utils.jid
import salt.state
import salt.payload
from salt.ext.six import string_types
@ -110,7 +111,7 @@ def running(concurrent=False):
).format(
data['fun'],
data['pid'],
salt.utils.jid_to_time(data['jid']),
salt.utils.jid.jid_to_time(data['jid']),
data['jid'],
)
ret.append(err)

4
salt/returners/carbon_return.py
View File

@ -56,7 +56,7 @@ import struct
import time
# Import salt libs
import salt.utils
import salt.utils.jid
import salt.returners
from salt.ext.six.moves import map
@ -238,4 +238,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/cassandra_return.py
View File

@ -24,7 +24,7 @@ from __future__ import absolute_import
import logging
# Import salt libs
import salt.utils
import salt.utils.jid
# Import third party libs
try:
@ -79,4 +79,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

5
salt/returners/couchbase_return.py
View File

@ -38,6 +38,7 @@ except ImportError:
# Import salt libs
import salt.utils
import salt.utils.jid
log = logging.getLogger(__name__)
@ -127,7 +128,7 @@ def prep_jid(nocache=False, passed_jid=None):
So do what you have to do to make sure that stays the case
'''
if passed_jid is None:
jid = salt.utils.gen_jid()
jid = salt.utils.jid.gen_jid()
else:
jid = passed_jid
@ -287,5 +288,5 @@ def _format_jid_instance(jid, job):
Return a properly formatted jid dict
'''
ret = _format_job_instance(job)
ret.update({'StartTime': salt.utils.jid_to_time(jid)})
ret.update({'StartTime': salt.utils.jid.jid_to_time(jid)})
return ret

4
salt/returners/couchdb_return.py
View File

@ -59,7 +59,7 @@ from salt.ext.six.moves.urllib.request import (
# pylint: enable=no-name-in-module,import-error
# Import Salt libs
import salt.utils
import salt.utils.jid
import salt.returners
log = logging.getLogger(__name__)
@ -364,4 +364,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/django_return.py
View File

@ -32,7 +32,7 @@ import logging
# Import Salt libraries
import salt.returners
import salt.utils
import salt.utils.jid
log = logging.getLogger(__name__)
@ -82,4 +82,4 @@ def prep_jid(nocache, passed_jid=None):
'''
Do any work necessary to prepare a JID, including sending a custom ID
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/elasticsearch_return.py
View File

@ -54,7 +54,7 @@ from __future__ import absolute_import
import datetime
# Import Salt libs
import salt.utils
import salt.utils.jid
__virtualname__ = 'elasticsearch'
@ -153,4 +153,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

3
salt/returners/etcd_return.py
View File

@ -54,6 +54,7 @@ except ImportError:
HAS_LIBS = False
import salt.utils
import salt.utils.jid
log = logging.getLogger(__name__)
@ -173,4 +174,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

5
salt/returners/local_cache.py
View File

@ -16,6 +16,7 @@ import hashlib
# Import salt libs
import salt.payload
import salt.utils
import salt.utils.jid
log = logging.getLogger(__name__)
@ -93,7 +94,7 @@ def _format_jid_instance(jid, job):
Format the jid correctly
'''
ret = _format_job_instance(job)
ret.update({'StartTime': salt.utils.jid_to_time(jid)})
ret.update({'StartTime': salt.utils.jid.jid_to_time(jid)})
return ret
@ -105,7 +106,7 @@ def prep_jid(nocache=False, passed_jid=None):
So do what you have to do to make sure that stays the case
'''
if passed_jid is None: # this can be a None of an empty string
jid = salt.utils.gen_jid()
jid = salt.utils.jid.gen_jid()
else:
jid = passed_jid

4
salt/returners/memcache_return.py
View File

@ -31,7 +31,7 @@ from __future__ import absolute_import
# Import python libs
import json
import logging
import salt.utils
import salt.utils.jid
import salt.returners
@ -98,7 +98,7 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()
def returner(ret):

4
salt/returners/mongo_future_return.py
View File

@ -43,7 +43,7 @@ from __future__ import absolute_import
import logging
# Import Salt libs
import salt.utils
import salt.utils.jid
import salt.returners
import salt.ext.six as six
@ -215,4 +215,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/mongo_return.py
View File

@ -39,7 +39,7 @@ from __future__ import absolute_import
import logging
# import Salt libs
import salt.utils
import salt.utils.jid
import salt.returners
import salt.ext.six as six
@ -163,4 +163,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/mysql.py
View File

@ -100,7 +100,7 @@ import logging
# Import salt libs
import salt.returners
import salt.utils
import salt.utils.jid
# Import third party libs
try:
@ -316,4 +316,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/odbc.py
View File

@ -114,7 +114,7 @@ from __future__ import absolute_import
import json
# Import Salt libs
import salt.utils
import salt.utils.jid
import salt.returners
# FIXME We'll need to handle this differently for Windows.
@ -305,4 +305,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/postgres.py
View File

@ -82,7 +82,7 @@ from __future__ import absolute_import
import json
# Import Salt libs
import salt.utils
import salt.utils.jid
import salt.returners
# Import third party libs
@ -277,4 +277,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

7
salt/returners/postgres_local_cache.py
View File

@ -77,6 +77,7 @@ import sys
# Import salt libs
import salt.utils
import salt.utils.jid
# Import third party libs
try:
import psycopg2
@ -148,7 +149,7 @@ def _format_jid_instance(jid, job):
Format the jid correctly
'''
ret = _format_job_instance(job)
ret.update({'StartTime': salt.utils.jid_to_time(jid)})
ret.update({'StartTime': salt.utils.jid.jid_to_time(jid)})
return ret
@ -156,7 +157,7 @@ def _gen_jid(cur):
'''
Generate an unique job id
'''
jid = salt.utils.gen_jid()
jid = salt.utils.jid.gen_jid()
sql = '''SELECT jid FROM jids WHERE jid = %s'''
cur.execute(sql, (jid,))
data = cur.fetchall()
@ -229,7 +230,7 @@ def save_load(jid, clear_load):
cur.execute(
sql, (
jid,
salt.utils.jid_to_time(jid),
salt.utils.jid.jid_to_time(jid),
str(clear_load.get("tgt_type")),
str(clear_load.get("cmd")),
str(clear_load.get("tgt")),

4
salt/returners/redis_return.py
View File

@ -34,7 +34,7 @@ from __future__ import absolute_import
import json
# Import Salt libs
import salt.utils
import salt.utils.jid
import salt.returners
# Import third party libs
@ -167,4 +167,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/sentry_return.py
View File

@ -29,7 +29,7 @@ from __future__ import absolute_import
import logging
# Import Salt libs
import salt.utils
import salt.utils.jid
try:
from raven import Client
@ -116,4 +116,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/smtp_return.py
View File

@ -72,7 +72,7 @@ import smtplib
from email.utils import formatdate
# Import Salt libs
import salt.utils
import salt.utils.jid
import salt.returners
try:
@ -188,4 +188,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/sqlite3_return.py
View File

@ -68,7 +68,7 @@ import json
import datetime
# Import Salt libs
import salt.utils
import salt.utils.jid
import salt.returners
# Better safe than sorry here. Even though sqlite3 is included in python
@ -276,4 +276,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/syslog_return.py
View File

@ -23,7 +23,7 @@ except ImportError:
HAS_SYSLOG = False
# Import Salt libs
import salt.utils
import salt.utils.jid
# Define the module's virtual name
__virtualname__ = 'syslog'
@ -46,4 +46,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

3
salt/runners/jobs.py
View File

@ -15,6 +15,7 @@ import os
import salt.client
import salt.payload
import salt.utils
import salt.utils.jid
import salt.minion
from salt.ext.six import string_types
@ -290,7 +291,7 @@ def _format_jid_instance(jid, job):
Helper to format jid instance
'''
ret = _format_job_instance(job)
ret.update({'StartTime': salt.utils.jid_to_time(jid)})
ret.update({'StartTime': salt.utils.jid.jid_to_time(jid)})
return ret

110
salt/utils/__init__.py
View File

@ -33,7 +33,6 @@ import types
import warnings
import string
import locale
from calendar import month_abbr as months
from salt.ext.six import string_types
from salt.ext.six.moves.urllib.parse import urlparse # pylint: disable=E0611
import salt.ext.six as six
@ -422,31 +421,6 @@ def list_files(directory):
return list(ret)
def jid_to_time(jid):
'''
Convert a salt job id into the time when the job was invoked
'''
jid = str(jid)
if len(jid) != 20:
return ''
year = jid[:4]
month = jid[4:6]
day = jid[6:8]
hour = jid[8:10]
minute = jid[10:12]
second = jid[12:14]
micro = jid[14:]
ret = '{0}, {1} {2} {3}:{4}:{5}.{6}'.format(year,
months[int(month)],
day,
hour,
minute,
second,
micro)
return ret
def gen_mac(prefix='AC:DE:48'):
'''
Generates a MAC address with the defined OUI prefix.
@ -550,90 +524,6 @@ def required_modules_error(name, docstring):
return msg.format(filename, ', '.join(modules))
def gen_jid():
'''
Generate a jid
'''
return '{0:%Y%m%d%H%M%S%f}'.format(datetime.datetime.now())
def prep_jid(cachedir, sum_type, user='root', nocache=False):
'''
Return a job id and prepare the job id directory
'''
salt.utils.warn_until(
'Boron',
'All job_cache management has been moved into the local_cache '
'returner, this util function will be removed-- please use '
'the returner'
)
jid = gen_jid()
jid_dir_ = jid_dir(jid, cachedir, sum_type)
if not os.path.isdir(jid_dir_):
if os.path.exists(jid_dir_):
# Somehow we ended up with a file at our jid destination.
# Delete it.
os.remove(jid_dir_)
os.makedirs(jid_dir_)
with fopen(os.path.join(jid_dir_, 'jid'), 'w+') as fn_:
fn_.write(jid)
if nocache:
with fopen(os.path.join(jid_dir_, 'nocache'), 'w+') as fn_:
fn_.write('')
else:
return prep_jid(cachedir, sum_type, user=user, nocache=nocache)
return jid
def jid_dir(jid, cachedir, sum_type):
'''
Return the jid_dir for the given job id
'''
salt.utils.warn_until(
'Boron',
'All job_cache management has been moved into the local_cache '
'returner, this util function will be removed-- please use '
'the returner'
)
jid = str(jid)
jhash = getattr(hashlib, sum_type)(jid).hexdigest()
return os.path.join(cachedir, 'jobs', jhash[:2], jhash[2:])
def jid_load(jid, cachedir, sum_type, serial='msgpack'):
'''
Return the load data for a given job id
'''
salt.utils.warn_until(
'Boron',
'Getting the load has been moved into the returner interface '
'please get the data from the master_job_cache '
)
_dir = jid_dir(jid, cachedir, sum_type)
load_fn = os.path.join(_dir, '.load.p')
if not os.path.isfile(load_fn):
return {}
serial = salt.payload.Serial(serial)
with fopen(load_fn, 'rb') as fp_:
return serial.load(fp_)
def is_jid(jid):
'''
Returns True if the passed in value is a job id
'''
if not isinstance(jid, string_types):
return False
if len(jid) != 20:
return False
try:
int(jid)
return True
except ValueError:
return False
def check_or_die(command):
'''
Simple convenience function for modules to use for gracefully blowing up

91
salt/utils/jid.py Normal file
View File

@ -0,0 +1,91 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import print_function
from calendar import month_abbr as months
import datetime
import hashlib
import os
import salt.utils
from salt.ext.six import string_types
def gen_jid():
'''
Generate a jid
'''
return '{0:%Y%m%d%H%M%S%f}'.format(datetime.datetime.now())
def is_jid(jid):
'''
Returns True if the passed in value is a job id
'''
if not isinstance(jid, string_types):
return False
if len(jid) != 20:
return False
try:
int(jid)
return True
except ValueError:
return False
def jid_dir(jid, cachedir, sum_type):
'''
Return the jid_dir for the given job id
'''
salt.utils.warn_until(
'Boron',
'All job_cache management has been moved into the local_cache '
'returner, this util function will be removed-- please use '
'the returner'
)
jid = str(jid)
jhash = getattr(hashlib, sum_type)(jid).hexdigest()
return os.path.join(cachedir, 'jobs', jhash[:2], jhash[2:])
def jid_load(jid, cachedir, sum_type, serial='msgpack'):
'''
Return the load data for a given job id
'''
salt.utils.warn_until(
'Boron',
'Getting the load has been moved into the returner interface '
'please get the data from the master_job_cache '
)
_dir = jid_dir(jid, cachedir, sum_type)
load_fn = os.path.join(_dir, '.load.p')
if not os.path.isfile(load_fn):
return {}
serial = salt.payload.Serial(serial)
with salt.utils.fopen(load_fn, 'rb') as fp_:
return serial.load(fp_)
def jid_to_time(jid):
'''
Convert a salt job id into the time when the job was invoked
'''
jid = str(jid)
if len(jid) != 20:
return ''
year = jid[:4]
month = jid[4:6]
day = jid[6:8]
hour = jid[8:10]
minute = jid[10:12]
second = jid[12:14]
micro = jid[14:]
ret = '{0}, {1} {2} {3}:{4}:{5}.{6}'.format(year,
months[int(month)],
day,
hour,
minute,
second,
micro)
return ret