valitydev/salt
14
0
Fork 0
mirror of https://github.com/valitydev/salt.git synced 2024-11-07 00:55:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #18938 from cachedout/jid_util

Browse Source 
Refactor jid utils out of utils/__init__
This commit is contained in:
Thomas S Hatch 2014-12-11 17:27:00 -07:00
commit b848e7ebe8
30 changed files with 236 additions and 239 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
salt/daemons/masterapi.py
View File

@ -38,6 +38,7 @@ import salt.utils.event
import salt.utils.verify
import salt.utils.minions
import salt.utils.gzip_util
import salt.utils.jid
from salt.pillar import git_pillar
from salt.utils.event import tagify
from salt.exceptions import SaltMasterError
@ -1135,7 +1136,7 @@ class LocalFuncs(object):
return dict(error=dict(name='TokenAuthenticationError',
message=msg))
jid = salt.utils.gen_jid()
jid = salt.utils.jid.gen_jid()
fun = load.pop('fun')
tag = tagify(jid, prefix='wheel')
data = {'fun': "wheel.{0}".format(fun),
@ -1205,7 +1206,7 @@ class LocalFuncs(object):
return dict(error=dict(name='EauthAuthenticationError',
message=msg))
jid = salt.utils.gen_jid()
jid = salt.utils.jid.gen_jid()
fun = load.pop('fun')
tag = tagify(jid, prefix='wheel')
data = {'fun': "wheel.{0}".format(fun),

5
salt/master.py
View File

@ -46,6 +46,7 @@ import salt.utils.minions
import salt.utils.gzip_util
import salt.utils.process
import salt.utils.zeromq
import salt.utils.jid
from salt.defaults import DEFAULT_TARGET_DELIM
from salt.utils.debug import enable_sigusr1_handler, enable_sigusr2_handler, inspect_stack
from salt.utils.event import tagify
@ -2017,7 +2018,7 @@ class ClearFuncs(object):
else:
token = self.loadauth.get_tok(clear_load['token'])
jid = salt.utils.gen_jid()
jid = salt.utils.jid.gen_jid()
fun = clear_load.pop('fun')
tag = tagify(jid, prefix='wheel')
data = {'fun': "wheel.{0}".format(fun),
@ -2049,7 +2050,7 @@ class ClearFuncs(object):
eauth_error = self.process_eauth(clear_load, 'wheel')
if eauth_error:
return eauth_error
jid = salt.utils.gen_jid()
jid = salt.utils.jid.gen_jid()
fun = clear_load.pop('fun')
tag = tagify(jid, prefix='wheel')
data = {'fun': "wheel.{0}".format(fun),

5
salt/minion.py
View File

@ -63,6 +63,7 @@ import salt.crypt
import salt.loader
import salt.payload
import salt.utils
import salt.utils.jid
import salt.utils.args
import salt.utils.event
import salt.utils.minion
@ -2107,7 +2108,7 @@ class Syndic(Minion):
self.event_forward_timeout = (
time.time() + self.opts['syndic_event_forward_timeout']
)
if salt.utils.is_jid(event['tag']) and 'return' in event['data']:
if salt.utils.jid.is_jid(event['tag']) and 'return' in event['data']:
if 'jid' not in event['data']:
# Not a job return
continue
@ -2344,7 +2345,7 @@ class MultiSyndic(MinionBase):
self.event_forward_timeout = (
time.time() + self.opts['syndic_event_forward_timeout']
)
if salt.utils.is_jid(event['tag']) and 'return' in event['data']:
if salt.utils.jid.is_jid(event['tag']) and 'return' in event['data']:
if 'jid' not in event['data']:
# Not a job return
continue

3
salt/modules/state.py
View File

@ -18,6 +18,7 @@ import tempfile
# Import salt libs
import salt.config
import salt.utils
import salt.utils.jid
import salt.state
import salt.payload
from salt.ext.six import string_types
@ -110,7 +111,7 @@ def running(concurrent=False):
).format(
data['fun'],
data['pid'],
salt.utils.jid_to_time(data['jid']),
salt.utils.jid.jid_to_time(data['jid']),
data['jid'],
)
ret.append(err)

4
salt/returners/carbon_return.py
View File

@ -56,7 +56,7 @@ import struct
import time
# Import salt libs
import salt.utils
import salt.utils.jid
import salt.returners
from salt.ext.six.moves import map
@ -238,4 +238,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/cassandra_return.py
View File

@ -24,7 +24,7 @@ from __future__ import absolute_import
import logging
# Import salt libs
import salt.utils
import salt.utils.jid
# Import third party libs
try:
@ -79,4 +79,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

5
salt/returners/couchbase_return.py
View File

@ -38,6 +38,7 @@ except ImportError:
# Import salt libs
import salt.utils
import salt.utils.jid
log = logging.getLogger(__name__)
@ -127,7 +128,7 @@ def prep_jid(nocache=False, passed_jid=None):
So do what you have to do to make sure that stays the case
'''
if passed_jid is None:
jid = salt.utils.gen_jid()
jid = salt.utils.jid.gen_jid()
else:
jid = passed_jid
@ -287,5 +288,5 @@ def _format_jid_instance(jid, job):
Return a properly formatted jid dict
'''
ret = _format_job_instance(job)
ret.update({'StartTime': salt.utils.jid_to_time(jid)})
ret.update({'StartTime': salt.utils.jid.jid_to_time(jid)})
return ret

4
salt/returners/couchdb_return.py
View File

@ -59,7 +59,7 @@ from salt.ext.six.moves.urllib.request import (
# pylint: enable=no-name-in-module,import-error
# Import Salt libs
import salt.utils
import salt.utils.jid
import salt.returners
log = logging.getLogger(__name__)
@ -364,4 +364,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/django_return.py
View File

@ -32,7 +32,7 @@ import logging
# Import Salt libraries
import salt.returners
import salt.utils
import salt.utils.jid
log = logging.getLogger(__name__)
@ -82,4 +82,4 @@ def prep_jid(nocache, passed_jid=None):
'''
Do any work necessary to prepare a JID, including sending a custom ID
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/elasticsearch_return.py
View File

@ -54,7 +54,7 @@ from __future__ import absolute_import
import datetime
# Import Salt libs
import salt.utils
import salt.utils.jid
__virtualname__ = 'elasticsearch'
@ -153,4 +153,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

3
salt/returners/etcd_return.py
View File

@ -54,6 +54,7 @@ except ImportError:
HAS_LIBS = False
import salt.utils
import salt.utils.jid
log = logging.getLogger(__name__)
@ -173,4 +174,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

5
salt/returners/local_cache.py
View File

@ -16,6 +16,7 @@ import hashlib
# Import salt libs
import salt.payload
import salt.utils
import salt.utils.jid
log = logging.getLogger(__name__)
@ -93,7 +94,7 @@ def _format_jid_instance(jid, job):
Format the jid correctly
'''
ret = _format_job_instance(job)
ret.update({'StartTime': salt.utils.jid_to_time(jid)})
ret.update({'StartTime': salt.utils.jid.jid_to_time(jid)})
return ret
@ -105,7 +106,7 @@ def prep_jid(nocache=False, passed_jid=None):
So do what you have to do to make sure that stays the case
'''
if passed_jid is None: # this can be a None of an empty string
jid = salt.utils.gen_jid()
jid = salt.utils.jid.gen_jid()
else:
jid = passed_jid

4
salt/returners/memcache_return.py
View File

@ -31,7 +31,7 @@ from __future__ import absolute_import
# Import python libs
import json
import logging
import salt.utils
import salt.utils.jid
import salt.returners
@ -98,7 +98,7 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()
def returner(ret):

4
salt/returners/mongo_future_return.py
View File

@ -43,7 +43,7 @@ from __future__ import absolute_import
import logging
# Import Salt libs
import salt.utils
import salt.utils.jid
import salt.returners
import salt.ext.six as six
@ -215,4 +215,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/mongo_return.py
View File

@ -39,7 +39,7 @@ from __future__ import absolute_import
import logging
# import Salt libs
import salt.utils
import salt.utils.jid
import salt.returners
import salt.ext.six as six
@ -163,4 +163,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/mysql.py
View File

@ -100,7 +100,7 @@ import logging
# Import salt libs
import salt.returners
import salt.utils
import salt.utils.jid
# Import third party libs
try:
@ -316,4 +316,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/odbc.py
View File

@ -114,7 +114,7 @@ from __future__ import absolute_import
import json
# Import Salt libs
import salt.utils
import salt.utils.jid
import salt.returners
# FIXME We'll need to handle this differently for Windows.
@ -305,4 +305,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/postgres.py
View File

@ -82,7 +82,7 @@ from __future__ import absolute_import
import json
# Import Salt libs
import salt.utils
import salt.utils.jid
import salt.returners
# Import third party libs
@ -277,4 +277,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

7
salt/returners/postgres_local_cache.py
View File

@ -77,6 +77,7 @@ import sys
# Import salt libs
import salt.utils
import salt.utils.jid
# Import third party libs
try:
import psycopg2
@ -148,7 +149,7 @@ def _format_jid_instance(jid, job):
Format the jid correctly
'''
ret = _format_job_instance(job)
ret.update({'StartTime': salt.utils.jid_to_time(jid)})
ret.update({'StartTime': salt.utils.jid.jid_to_time(jid)})
return ret
@ -156,7 +157,7 @@ def _gen_jid(cur):
'''
Generate an unique job id
'''
jid = salt.utils.gen_jid()
jid = salt.utils.jid.gen_jid()
sql = '''SELECT jid FROM jids WHERE jid = %s'''
cur.execute(sql, (jid,))
data = cur.fetchall()
@ -229,7 +230,7 @@ def save_load(jid, clear_load):
cur.execute(
sql, (
jid,
salt.utils.jid_to_time(jid),
salt.utils.jid.jid_to_time(jid),
str(clear_load.get("tgt_type")),
str(clear_load.get("cmd")),
str(clear_load.get("tgt")),

4
salt/returners/redis_return.py
View File

@ -34,7 +34,7 @@ from __future__ import absolute_import
import json
# Import Salt libs
import salt.utils
import salt.utils.jid
import salt.returners
# Import third party libs
@ -167,4 +167,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/sentry_return.py
View File

@ -29,7 +29,7 @@ from __future__ import absolute_import
import logging
# Import Salt libs
import salt.utils
import salt.utils.jid
try:
from raven import Client
@ -116,4 +116,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/smtp_return.py
View File

@ -72,7 +72,7 @@ import smtplib
from email.utils import formatdate
# Import Salt libs
import salt.utils
import salt.utils.jid
import salt.returners
try:
@ -188,4 +188,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/sqlite3_return.py
View File

@ -68,7 +68,7 @@ import json
import datetime
# Import Salt libs
import salt.utils
import salt.utils.jid
import salt.returners
# Better safe than sorry here. Even though sqlite3 is included in python
@ -276,4 +276,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

4
salt/returners/syslog_return.py
View File

@ -23,7 +23,7 @@ except ImportError:
HAS_SYSLOG = False
# Import Salt libs
import salt.utils
import salt.utils.jid
# Define the module's virtual name
__virtualname__ = 'syslog'
@ -46,4 +46,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument
'''
Do any work necessary to prepare a JID, including sending a custom id
'''
return passed_jid if passed_jid is not None else salt.utils.gen_jid()
return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()

3
salt/runners/jobs.py
View File

@ -15,6 +15,7 @@ import os
import salt.client
import salt.payload
import salt.utils
import salt.utils.jid
import salt.minion
from salt.ext.six import string_types
@ -290,7 +291,7 @@ def _format_jid_instance(jid, job):
Helper to format jid instance
'''
ret = _format_job_instance(job)
ret.update({'StartTime': salt.utils.jid_to_time(jid)})
ret.update({'StartTime': salt.utils.jid.jid_to_time(jid)})
return ret

183
salt/utils/__init__.py
View File

@ -33,7 +33,6 @@ import types
import warnings
import string
import locale
from calendar import month_abbr as months
from salt.ext.six import string_types
from salt.ext.six.moves.urllib.parse import urlparse # pylint: disable=E0611
import salt.ext.six as six
@ -97,7 +96,6 @@ except ImportError:
# Import salt libs
from salt.defaults import DEFAULT_TARGET_DELIM
import salt.log
import salt.payload
import salt.version
import salt.defaults.exitcodes
from salt.utils.decorators import memoize as real_memoize
@ -422,31 +420,6 @@ def list_files(directory):
return list(ret)
def jid_to_time(jid):
'''
Convert a salt job id into the time when the job was invoked
'''
jid = str(jid)
if len(jid) != 20:
return ''
year = jid[:4]
month = jid[4:6]
day = jid[6:8]
hour = jid[8:10]
minute = jid[10:12]
second = jid[12:14]
micro = jid[14:]
ret = '{0}, {1} {2} {3}:{4}:{5}.{6}'.format(year,
months[int(month)],
day,
hour,
minute,
second,
micro)
return ret
def gen_mac(prefix='AC:DE:48'):
'''
Generates a MAC address with the defined OUI prefix.
@ -550,90 +523,6 @@ def required_modules_error(name, docstring):
return msg.format(filename, ', '.join(modules))
def gen_jid():
'''
Generate a jid
'''
return '{0:%Y%m%d%H%M%S%f}'.format(datetime.datetime.now())
def prep_jid(cachedir, sum_type, user='root', nocache=False):
'''
Return a job id and prepare the job id directory
'''
salt.utils.warn_until(
'Boron',
'All job_cache management has been moved into the local_cache '
'returner, this util function will be removed-- please use '
'the returner'
)
jid = gen_jid()
jid_dir_ = jid_dir(jid, cachedir, sum_type)
if not os.path.isdir(jid_dir_):
if os.path.exists(jid_dir_):
# Somehow we ended up with a file at our jid destination.
# Delete it.
os.remove(jid_dir_)
os.makedirs(jid_dir_)
with fopen(os.path.join(jid_dir_, 'jid'), 'w+') as fn_:
fn_.write(jid)
if nocache:
with fopen(os.path.join(jid_dir_, 'nocache'), 'w+') as fn_:
fn_.write('')
else:
return prep_jid(cachedir, sum_type, user=user, nocache=nocache)
return jid
def jid_dir(jid, cachedir, sum_type):
'''
Return the jid_dir for the given job id
'''
salt.utils.warn_until(
'Boron',
'All job_cache management has been moved into the local_cache '
'returner, this util function will be removed-- please use '
'the returner'
)
jid = str(jid)
jhash = getattr(hashlib, sum_type)(jid).hexdigest()
return os.path.join(cachedir, 'jobs', jhash[:2], jhash[2:])
def jid_load(jid, cachedir, sum_type, serial='msgpack'):
'''
Return the load data for a given job id
'''
salt.utils.warn_until(
'Boron',
'Getting the load has been moved into the returner interface '
'please get the data from the master_job_cache '
)
_dir = jid_dir(jid, cachedir, sum_type)
load_fn = os.path.join(_dir, '.load.p')
if not os.path.isfile(load_fn):
return {}
serial = salt.payload.Serial(serial)
with fopen(load_fn, 'rb') as fp_:
return serial.load(fp_)
def is_jid(jid):
'''
Returns True if the passed in value is a job id
'''
if not isinstance(jid, string_types):
return False
if len(jid) != 20:
return False
try:
int(jid)
return True
except ValueError:
return False
def check_or_die(command):
'''
Simple convenience function for modules to use for gracefully blowing up
@ -2252,78 +2141,6 @@ def get_gid_list(user=None, include_default=True):
return sorted(set(gid_list))
def trim_dict(
data,
max_dict_bytes,
percent=50.0,
stepper_size=10,
replace_with='VALUE_TRIMMED',
is_msgpacked=False):
'''
Takes a dictionary and iterates over its keys, looking for
large values and replacing them with a trimmed string.
If after the first pass over dictionary keys, the dictionary
is not sufficiently small, the stepper_size will be increased
and the dictionary will be rescanned. This allows for progressive
scanning, removing large items first and only making additional
passes for smaller items if necessary.
This function uses msgpack to calculate the size of the dictionary
in question. While this might seem like unnecessary overhead, a
data structure in python must be serialized in order for sys.getsizeof()
to accurately return the items referenced in the structure.
Ex:
>>> salt.utils.trim_dict({'a': 'b', 'c': 'x' * 10000}, 100)
{'a': 'b', 'c': 'VALUE_TRIMMED'}
To improve performance, it is adviseable to pass in msgpacked
data structures instead of raw dictionaries. If a msgpack
structure is passed in, it will not be unserialized unless
necessary.
If a msgpack is passed in, it will be repacked if necessary
before being returned.
'''
serializer = salt.payload.Serial({'serial': 'msgpack'})
if is_msgpacked:
dict_size = sys.getsizeof(data)
else:
dict_size = sys.getsizeof(serializer.dumps(data))
if dict_size > max_dict_bytes:
if is_msgpacked:
data = serializer.loads(data)
while True:
percent = float(percent)
max_val_size = float(max_dict_bytes * (percent / 100))
try:
for key in data:
if sys.getsizeof(data[key]) > max_val_size:
data[key] = replace_with
percent = percent - stepper_size
max_val_size = float(max_dict_bytes * (percent / 100))
cur_dict_size = sys.getsizeof(serializer.dumps(data))
if cur_dict_size < max_dict_bytes:
if is_msgpacked: # Repack it
return serializer.dumps(data)
else:
return data
elif max_val_size == 0:
if is_msgpacked:
return serializer.dumps(data)
else:
return data
except ValueError:
pass
if is_msgpacked:
return serializer.dumps(data)
else:
return data
else:
return data
def total_seconds(td):
'''
Takes a timedelta and returns the total number of seconds

79
salt/utils/dicttrim.py Normal file
View File

@ -0,0 +1,79 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import print_function
import sys
import salt.payload
def trim_dict(
data,
max_dict_bytes,
percent=50.0,
stepper_size=10,
replace_with='VALUE_TRIMMED',
is_msgpacked=False):
'''
Takes a dictionary and iterates over its keys, looking for
large values and replacing them with a trimmed string.
If after the first pass over dictionary keys, the dictionary
is not sufficiently small, the stepper_size will be increased
and the dictionary will be rescanned. This allows for progressive
scanning, removing large items first and only making additional
passes for smaller items if necessary.
This function uses msgpack to calculate the size of the dictionary
in question. While this might seem like unnecessary overhead, a
data structure in python must be serialized in order for sys.getsizeof()
to accurately return the items referenced in the structure.
Ex:
>>> salt.utils.trim_dict({'a': 'b', 'c': 'x' * 10000}, 100)
{'a': 'b', 'c': 'VALUE_TRIMMED'}
To improve performance, it is adviseable to pass in msgpacked
data structures instead of raw dictionaries. If a msgpack
structure is passed in, it will not be unserialized unless
necessary.
If a msgpack is passed in, it will be repacked if necessary
before being returned.
'''
serializer = salt.payload.Serial({'serial': 'msgpack'})
if is_msgpacked:
dict_size = sys.getsizeof(data)
else:
dict_size = sys.getsizeof(serializer.dumps(data))
if dict_size > max_dict_bytes:
if is_msgpacked:
data = serializer.loads(data)
while True:
percent = float(percent)
max_val_size = float(max_dict_bytes * (percent / 100))
try:
for key in data:
if sys.getsizeof(data[key]) > max_val_size:
data[key] = replace_with
percent = percent - stepper_size
max_val_size = float(max_dict_bytes * (percent / 100))
cur_dict_size = sys.getsizeof(serializer.dumps(data))
if cur_dict_size < max_dict_bytes:
if is_msgpacked: # Repack it
return serializer.dumps(data)
else:
return data
elif max_val_size == 0:
if is_msgpacked:
return serializer.dumps(data)
else:
return data
except ValueError:
pass
if is_msgpacked:
return serializer.dumps(data)
else:
return data
else:
return data

3
salt/utils/event.py
View File

@ -71,6 +71,7 @@ import salt.payload
import salt.loader
import salt.utils
import salt.utils.cache
import salt.utils.dicttrim
import salt.utils.process
import salt.utils.zeromq
log = logging.getLogger(__name__)
@ -414,7 +415,7 @@ class SaltEvent(object):
data['_stamp'] = datetime.datetime.now().isoformat()
tagend = TAGEND
serialized_data = salt.utils.trim_dict(self.serial.dumps(data),
serialized_data = salt.utils.dicttrim.trim_dict(self.serial.dumps(data),
self.opts.get('max_event_size', 1048576),
is_msgpacked=True
)

91
salt/utils/jid.py Normal file
View File

@ -0,0 +1,91 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import print_function
from calendar import month_abbr as months
import datetime
import hashlib
import os
import salt.utils
from salt.ext.six import string_types
def gen_jid():
'''
Generate a jid
'''
return '{0:%Y%m%d%H%M%S%f}'.format(datetime.datetime.now())
def is_jid(jid):
'''
Returns True if the passed in value is a job id
'''
if not isinstance(jid, string_types):
return False
if len(jid) != 20:
return False
try:
int(jid)
return True
except ValueError:
return False
def jid_dir(jid, cachedir, sum_type):
'''
Return the jid_dir for the given job id
'''
salt.utils.warn_until(
'Boron',
'All job_cache management has been moved into the local_cache '
'returner, this util function will be removed-- please use '
'the returner'
)
jid = str(jid)
jhash = getattr(hashlib, sum_type)(jid).hexdigest()
return os.path.join(cachedir, 'jobs', jhash[:2], jhash[2:])
def jid_load(jid, cachedir, sum_type, serial='msgpack'):
'''
Return the load data for a given job id
'''
salt.utils.warn_until(
'Boron',
'Getting the load has been moved into the returner interface '
'please get the data from the master_job_cache '
)
_dir = jid_dir(jid, cachedir, sum_type)
load_fn = os.path.join(_dir, '.load.p')
if not os.path.isfile(load_fn):
return {}
serial = salt.payload.Serial(serial)
with salt.utils.fopen(load_fn, 'rb') as fp_:
return serial.load(fp_)
def jid_to_time(jid):
'''
Convert a salt job id into the time when the job was invoked
'''
jid = str(jid)
if len(jid) != 20:
return ''
year = jid[:4]
month = jid[4:6]
day = jid[6:8]
hour = jid[8:10]
minute = jid[10:12]
second = jid[12:14]
micro = jid[14:]
ret = '{0}, {1} {2} {3}:{4}:{5}.{6}'.format(year,
months[int(month)],
day,
hour,
minute,
second,
micro)
return ret

14
tests/unit/utils/utils_test.py
View File

@ -59,11 +59,11 @@ class UtilsTestCase(TestCase):
def test_jid_to_time(self):
test_jid = 20131219110700123489
expected_jid = '2013, Dec 19 11:07:00.123489'
self.assertEqual(utils.jid_to_time(test_jid), expected_jid)
self.assertEqual(utils.jid.jid_to_time(test_jid), expected_jid)
# Test incorrect lengths
incorrect_jid_length = 2012
self.assertEqual(utils.jid_to_time(incorrect_jid_length), '')
self.assertEqual(utils.jid.jid_to_time(incorrect_jid_length), '')
@skipIf(NO_MOCK, NO_MOCK_REASON)
@patch('random.randint', return_value=1)
@ -86,14 +86,14 @@ class UtilsTestCase(TestCase):
expected_jid_dir = '/tmp/cachdir/jobs/69/fda308ccfa70d8296345e6509de136'
ret = utils.jid_dir(test_jid, test_cache_dir, test_hash_type)
ret = utils.jid.jid_dir(test_jid, test_cache_dir, test_hash_type)
self.assertEqual(ret, expected_jid_dir)
def test_is_jid(self):
self.assertTrue(utils.is_jid('20131219110700123489')) # Valid JID
self.assertFalse(utils.is_jid(20131219110700123489)) # int
self.assertFalse(utils.is_jid('2013121911070012348911111')) # Wrong length
self.assertTrue(utils.jid.is_jid('20131219110700123489')) # Valid JID
self.assertFalse(utils.jid.is_jid(20131219110700123489)) # int
self.assertFalse(utils.jid.is_jid('2013121911070012348911111')) # Wrong length
@skipIf(NO_MOCK, NO_MOCK_REASON)
@patch('salt.utils.is_windows', return_value=False)
@ -696,7 +696,7 @@ class UtilsTestCase(TestCase):
now = datetime.datetime(2002, 12, 25, 12, 00, 00, 00)
with patch('datetime.datetime'):
datetime.datetime.now.return_value = now
ret = utils.gen_jid()
ret = utils.jid.gen_jid()
self.assertEqual(ret, '20021225120000000000')
@skipIf(NO_MOCK, NO_MOCK_REASON)