valitydev/salt
14
0
Fork 0
mirror of https://github.com/valitydev/salt.git synced 2024-11-08 17:33:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'msgpack' into develop

Browse Source
This commit is contained in:
Thomas S Hatch 2011-12-13 22:12:34 -07:00
commit 251c238e7d
21 changed files with 9046 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
salt/__init__.py
View File

@ -10,10 +10,13 @@ import optparse
import os
import sys
# Import salt libs
import salt.config
import salt.utils.verify
# Import salt libs, the try block bypasses an issue at build time so that c
# modules don't cause the build to fail
try:
import salt.config
import salt.utils.verify
except ImportError:
pass
def verify_env(dirs):
'''

14
salt/client.py
View File

@ -26,7 +26,6 @@ The data structure needs to be:
# small, and only start with the ability to execute salt commands locally.
# This means that the primary client to build is, the LocalClient
import cPickle as pickle
import datetime
import glob
import os
@ -72,6 +71,7 @@ class LocalClient(object):
'''
def __init__(self, c_path='/etc/salt/master'):
self.opts = salt.config.master_config(c_path)
self.serial = salt.payload.Serial(self.opts)
self.key = self.__read_master_key()
def __read_master_key(self):
@ -200,7 +200,7 @@ class LocalClient(object):
continue
while fn_ not in ret:
try:
ret[fn_] = pickle.load(open(retp, 'r'))
ret[fn_] = self.serial.load(open(retp, 'r'))
except:
pass
if ret and start == 999999999999:
@ -239,10 +239,10 @@ class LocalClient(object):
continue
while fn_ not in ret:
try:
ret_data = pickle.load(open(retp, 'r'))
ret_data = self.serial.load(open(retp, 'r'))
ret[fn_] = {'ret': ret_data}
if os.path.isfile(outp):
ret[fn_]['out'] = pickle.load(open(outp, 'r'))
ret[fn_]['out'] = self.serial.load(open(outp, 'r'))
except:
pass
if ret and start == 999999999999:
@ -269,7 +269,7 @@ class LocalClient(object):
loadp = os.path.join(jid_dir, '.load.p')
if os.path.isfile(loadp):
try:
load = pickle.load(open(loadp, 'r'))
load = self.serial.load(open(loadp, 'r'))
if load['fun'] == cmd:
# We found a match! Add the return values
ret[jid] = {}
@ -278,7 +278,7 @@ class LocalClient(object):
retp = os.path.join(host_dir, 'return.p')
if not os.path.isfile(retp):
continue
ret[jid][host] = pickle.load(open(retp))
ret[jid][host] = self.serial.load(open(retp))
except:
continue
else:
@ -370,7 +370,7 @@ class LocalClient(object):
payload = None
for ind in range(100):
try:
payload = salt.payload.unpackage(
payload = self.serial.loads(
socket.recv(
zmq.NOBLOCK
)

1
salt/config.py
View File

@ -136,6 +136,7 @@ def master_config(path):
'log_granular_levels': {},
'cluster_masters': [],
'cluster_mode': 'paranoid',
'serial': 'msgpack',
}
load_config(opts, path, 'SALT_MASTER_CONFIG')

23
salt/crypt.py
View File

@ -5,7 +5,6 @@ authenticating peers
'''
# Import python libs
import cPickle as pickle
import hashlib
import hmac
import logging
@ -98,6 +97,7 @@ class Auth(object):
'''
def __init__(self, opts):
self.opts = opts
self.serial = salt.payload.Serial(self.opts)
self.rsa_path = os.path.join(self.opts['pki_dir'], 'minion.pem')
if 'syndic_master' in self.opts:
self.mpub = 'syndic_master.pub'
@ -188,9 +188,9 @@ class Auth(object):
context = zmq.Context()
socket = context.socket(zmq.REQ)
socket.connect(self.opts['master_uri'])
payload = salt.payload.package(self.minion_sign_in_payload())
payload = self.serial.dumps(self.minion_sign_in_payload())
socket.send(payload)
payload = salt.payload.unpackage(socket.recv())
payload = self.serial.loads(socket.recv())
if 'load' in payload:
if 'ret' in payload['load']:
if not payload['load']['ret']:
@ -244,9 +244,10 @@ class Crypticle(object):
AES_BLOCK_SIZE = 16
SIG_SIZE = hashlib.sha256().digest_size
def __init__(self, key_string, key_size=192):
def __init__(self, opts, key_string, key_size=192):
self.keys = self.extract_keys(key_string, key_size)
self.key_size = key_size
self.serial = salt.payload.Serial(opts)
@classmethod
def generate_key_string(cls, key_size=192):
@ -288,21 +289,21 @@ class Crypticle(object):
data = cypher.decrypt(data)
return data[:-ord(data[-1])]
def dumps(self, obj, pickler=pickle):
def dumps(self, obj):
'''
pickle and encrypt a python object
Serialize and encrypt a python object
'''
return self.encrypt(self.PICKLE_PAD + pickler.dumps(obj))
return self.encrypt(self.PICKLE_PAD + self.serial.dumps(obj))
def loads(self, data, pickler=pickle):
def loads(self, data):
'''
decrypt and un-pickle a python object
Decrypt and un-serialize a python object
'''
data = self.decrypt(data)
# simple integrity check to verify that we got meaningful data
if not data.startswith(self.PICKLE_PAD):
return {}
return pickler.loads(data[len(self.PICKLE_PAD):])
return self.serial.loads(data[len(self.PICKLE_PAD):])
class SAuth(Auth):
@ -326,7 +327,7 @@ class SAuth(Auth):
print 'Failed to authenticate with the master, verify that this'\
+ ' minion\'s public key has been accepted on the salt master'
sys.exit(2)
return Crypticle(creds['aes'])
return Crypticle(self.opts, creds['aes'])
def gen_token(self, clear_tok):
'''

36
salt/master.py
View File

@ -4,7 +4,6 @@ involves preparing the three listeners and the workers needed by the master.
'''
# Import python modules
import cPickle as pickle
import datetime
import hashlib
import logging
@ -29,20 +28,21 @@ import salt.utils
log = logging.getLogger(__name__)
def prep_jid(cachedir, load):
def prep_jid(opts, load):
'''
Parses the job return directory, generates a job id and sets up the
job id directory.
'''
serial = salt.payload.Serial(opts)
jid_root = os.path.join(cachedir, 'jobs')
jid = "{0:%Y%m%d%H%M%S%f}".format(datetime.datetime.now())
jid_dir = os.path.join(jid_root, jid)
if not os.path.isdir(jid_dir):
os.makedirs(jid_dir)
pickle.dump(load, open(os.path.join(jid_dir, '.load.p'), 'w+'))
serial.dump(load, open(os.path.join(jid_dir, '.load.p'), 'w+'))
else:
return prep_jid(load)
return prep_jid(cachedir, load)
return jid
@ -63,7 +63,7 @@ class SMaster(object):
'''
Return the crypticle used for AES
'''
return salt.crypt.Crypticle(self.opts['aes'])
return salt.crypt.Crypticle(self.opts, self.opts['aes'])
def __prep_key(self):
'''
@ -230,6 +230,7 @@ class MWorker(multiprocessing.Process):
clear_funcs):
multiprocessing.Process.__init__(self)
self.opts = opts
self.serial = salt.payload.Serial(opts)
self.crypticle = crypticle
self.aes_funcs = aes_funcs
self.clear_funcs = clear_funcs
@ -248,8 +249,8 @@ class MWorker(multiprocessing.Process):
while True:
package = socket.recv()
payload = salt.payload.unpackage(package)
ret = salt.payload.package(self._handle_payload(payload))
payload = self.serial.loads(package)
ret = self.serial.dumps(self._handle_payload(payload))
socket.send(ret)
def _handle_payload(self, payload):
@ -257,6 +258,11 @@ class MWorker(multiprocessing.Process):
The _handle_payload method is the key method used to figure out what
needs to be done with communication to the server
'''
try:
key = payload['enc']
load = payload['load']
except KeyError:
return ''
return {'aes': self._handle_aes,
'pub': self._handle_pub,
'clear': self._handle_clear}[payload['enc']](payload['load'])
@ -303,6 +309,7 @@ class AESFuncs(object):
#
def __init__(self, opts, crypticle):
self.opts = opts
self.serial = salt.payload.Serial(opts)
self.crypticle = crypticle
# Make a client
self.local = salt.client.LocalClient(self.opts['conf_file'])
@ -425,10 +432,10 @@ class AESFuncs(object):
hn_dir = os.path.join(jid_dir, load['id'])
if not os.path.isdir(hn_dir):
os.makedirs(hn_dir)
pickle.dump(load['return'],
self.serial.dump(load['return'],
open(os.path.join(hn_dir, 'return.p'), 'w+'))
if 'out' in load:
pickle.dump(load['out'],
self.serial.dump(load['out'],
open(os.path.join(hn_dir, 'out.p'), 'w+'))
def _syndic_return(self, load):
@ -498,7 +505,7 @@ class AESFuncs(object):
if not good:
return {}
# Set up the publication payload
jid = prep_jid(self.opts['cachedir'], clear_load)
jid = prep_jid(self.opts, clear_load)
payload = {'enc': 'aes'}
load = {
'fun': clear_load['fun'],
@ -523,7 +530,7 @@ class AESFuncs(object):
os.path.join(self.opts['sock_dir'], 'publish_pull.ipc')
)
pub_sock.connect(pull_uri)
pub_sock.send(salt.payload.package(payload))
pub_sock.send(self.serial.dumps(payload))
# Run the client get_returns method
return self.local.get_returns(
jid,
@ -562,6 +569,7 @@ class ClearFuncs(object):
# _auth
def __init__(self, opts, key, master_key, crypticle):
self.opts = opts
self.serial = salt.payload.Serial(opts)
self.key = key
self.master_key = master_key
self.crypticle = crypticle
@ -608,7 +616,7 @@ class ClearFuncs(object):
# 1. Verify that the key we are receiving matches the stored key
# 2. Store the key if it is not there
# 3. make an rsa key with the pub key
# 4. encrypt the aes key as an encrypted pickle
# 4. encrypt the aes key as an encrypted salt.payload
# 5. package the return and return it
log.info('Authentication request from %(id)s', load)
pubfn = os.path.join(self.opts['pki_dir'],
@ -696,7 +704,7 @@ class ClearFuncs(object):
if not os.path.isdir(jid_dir):
os.makedirs(jid_dir)
# Save the invocation information
pickle.dump(clear_load, open(os.path.join(jid_dir, '.load.p'), 'w+'))
self.serial.dump(clear_load, open(os.path.join(jid_dir, '.load.p'), 'w+'))
# Set up the payload
payload = {'enc': 'aes'}
load = {
@ -718,6 +726,6 @@ class ClearFuncs(object):
os.path.join(self.opts['sock_dir'], 'publish_pull.ipc')
)
pub_sock.connect(pull_uri)
pub_sock.send(salt.payload.package(payload))
pub_sock.send(self.serial.dumps(payload))
return {'enc': 'clear',
'load': {'jid': clear_load['jid']}}

31
salt/minion.py
View File

@ -25,6 +25,7 @@ import salt.loader
import salt.modules
import salt.returners
import salt.utils
import salt.payload
log = logging.getLogger(__name__)
@ -77,6 +78,7 @@ class Minion(object):
Pass in the options dict
'''
self.opts = opts
self.serial = salt.payload.Serial(self.opts)
self.mod_opts = self.__prep_mod_opts()
self.functions, self.returners = self.__load_modules()
self.matcher = Matcher(self.opts, self.functions)
@ -290,7 +292,7 @@ class Minion(object):
except KeyError:
pass
payload['load'] = self.crypticle.dumps(load)
socket.send_pyobj(payload)
socket.send(self.serial.dumps(payload))
return socket.recv()
def authenticate(self):
@ -311,7 +313,7 @@ class Minion(object):
time.sleep(10)
self.aes = creds['aes']
self.publish_port = creds['publish_port']
self.crypticle = salt.crypt.Crypticle(self.aes)
self.crypticle = salt.crypt.Crypticle(self.opts, self.aes)
def passive_refresh(self):
'''
@ -349,7 +351,7 @@ class Minion(object):
while True:
payload = None
try:
payload = socket.recv_pyobj(1)
payload = self.serial.loads(socket.recv(1))
self._handle_payload(payload)
last = time.time()
except:
@ -369,7 +371,7 @@ class Minion(object):
while True:
payload = None
try:
payload = socket.recv_pyobj(1)
payload = self.serial(socket.recv(1))
self._handle_payload(payload)
except:
pass
@ -579,6 +581,7 @@ class FileClient(object):
'''
def __init__(self, opts):
self.opts = opts
self.serial = salt.payload.Serial(self.opts)
self.auth = salt.crypt.SAuth(opts)
self.socket = self.__get_socket()
@ -623,8 +626,8 @@ class FileClient(object):
else:
load['loc'] = fn_.tell()
payload['load'] = self.auth.crypticle.dumps(load)
self.socket.send_pyobj(payload)
data = self.auth.crypticle.loads(self.socket.recv_pyobj())
self.socket.send(self.serial.dumps(payload))
data = self.auth.crypticle.loads(self.serial.loads(self.socket.recv()))
if not data['data']:
break
if not fn_:
@ -686,8 +689,8 @@ class FileClient(object):
load = {'env': env,
'cmd': '_file_list'}
payload['load'] = self.auth.crypticle.dumps(load)
self.socket.send_pyobj(payload)
return self.auth.crypticle.loads(self.socket.recv_pyobj())
self.socket.send(self.serial.dumps(payload))
return self.auth.crypticle.loads(self.serial.loads(self.socket.recv()))
def hash_file(self, path, env='base'):
'''
@ -701,8 +704,8 @@ class FileClient(object):
'env': env,
'cmd': '_file_hash'}
payload['load'] = self.auth.crypticle.dumps(load)
self.socket.send_pyobj(payload)
return self.auth.crypticle.loads(self.socket.recv_pyobj())
self.socket.send(self.serial.dumps(payload))
return self.auth.crypticle.loads(self.serial.loads(self.socket.recv()))
def list_env(self, path, env='base'):
'''
@ -712,8 +715,8 @@ class FileClient(object):
load = {'env': env,
'cmd': '_file_list'}
payload['load'] = self.auth.crypticle.dumps(load)
self.socket.send_pyobj(payload)
return self.auth.crypticle.loads(self.socket.recv_pyobj())
self.socket.send(self.serial.dumps(payload))
return self.auth.crypticle.loads(self.serial.loads(self.socket.recv()))
def get_state(self, sls, env):
'''
@ -736,5 +739,5 @@ class FileClient(object):
payload = {'enc': 'aes'}
load = {'cmd': '_master_opts'}
payload['load'] = self.auth.crypticle.dumps(load)
self.socket.send_pyobj(payload)
return self.auth.crypticle.loads(self.socket.recv_pyobj())
self.socket.send(self.serial.dumps(payload))
return self.auth.crypticle.loads(self.serial.loads(self.socket.recv()))

6
salt/modules/publish.py
View File

@ -5,6 +5,7 @@ Publish a command from a minion to a target
import zmq
import salt.crypt
import salt.payload
def _get_socket():
@ -35,6 +36,7 @@ def publish(tgt, fun, arg=None, expr_form='glob', returner=''):
salt system.example.com publish.publish '*' cmd.run 'ls -la /tmp'
'''
serial = salt.payload.Serial(__opts__)
if fun == 'publish.publish':
# Need to log something here
return {}
@ -55,5 +57,5 @@ def publish(tgt, fun, arg=None, expr_form='glob', returner=''):
'id': __opts__['id']}
payload['load'] = auth.crypticle.dumps(load)
socket = _get_socket()
socket.send_pyobj(payload)
return auth.crypticle.loads(socket.recv_pyobj())
socket.send(serial.dumps(payload))
return auth.crypticle.loads(serial.loads(socket.recv()))

14
salt/msgpack/COPYING Normal file
View File

@ -0,0 +1,14 @@
Copyright (C) 2008-2011 INADA Naoki <songofacandy@gmail.com>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

11
salt/msgpack/__init__.py Normal file
View File

@ -0,0 +1,11 @@
# coding: utf-8
from salt.msgpack.__version__ import *
from salt.msgpack._msgpack import *
# alias for compatibility to simplejson/marshal/pickle.
load = unpack
loads = unpackb
dump = pack
dumps = packb

1
salt/msgpack/__version__.py Normal file
View File

@ -0,0 +1 @@
version = (0, 1, 10)

6856
salt/msgpack/_msgpack.c Normal file
View File

File diff suppressed because it is too large Load Diff

408
salt/msgpack/_msgpack.pyx Normal file
View File

@ -0,0 +1,408 @@
# coding: utf-8
from cpython cimport *
cdef extern from "Python.h":
ctypedef char* const_char_ptr "const char*"
ctypedef char* const_void_ptr "const void*"
ctypedef struct PyObject
cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1
from libc.stdlib cimport *
from libc.string cimport *
import gc
_gc_disable = gc.disable
_gc_enable = gc.enable
cdef extern from "pack.h":
struct msgpack_packer:
char* buf
size_t length
size_t buf_size
int msgpack_pack_int(msgpack_packer* pk, int d)
int msgpack_pack_nil(msgpack_packer* pk)
int msgpack_pack_true(msgpack_packer* pk)
int msgpack_pack_false(msgpack_packer* pk)
int msgpack_pack_long(msgpack_packer* pk, long d)
int msgpack_pack_long_long(msgpack_packer* pk, long long d)
int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d)
int msgpack_pack_double(msgpack_packer* pk, double d)
int msgpack_pack_array(msgpack_packer* pk, size_t l)
int msgpack_pack_map(msgpack_packer* pk, size_t l)
int msgpack_pack_raw(msgpack_packer* pk, size_t l)
int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
cdef int DEFAULT_RECURSE_LIMIT=511
cdef class Packer(object):
"""MessagePack Packer
usage:
packer = Packer()
astream.write(packer.pack(a))
astream.write(packer.pack(b))
"""
cdef msgpack_packer pk
cdef object _default
cdef object _bencoding
cdef object _berrors
cdef char *encoding
cdef char *unicode_errors
def __cinit__(self):
cdef int buf_size = 1024*1024
self.pk.buf = <char*> malloc(buf_size);
if self.pk.buf == NULL:
raise MemoryError("Unable to allocate internal buffer.")
self.pk.buf_size = buf_size
self.pk.length = 0
def __init__(self, default=None, encoding='utf-8', unicode_errors='strict'):
if default is not None:
if not PyCallable_Check(default):
raise TypeError("default must be a callable.")
self._default = default
if encoding is None:
self.encoding = NULL
self.unicode_errors = NULL
else:
if isinstance(encoding, unicode):
self._bencoding = encoding.encode('ascii')
else:
self._bencoding = encoding
self.encoding = PyBytes_AsString(self._bencoding)
if isinstance(unicode_errors, unicode):
self._berrors = unicode_errors.encode('ascii')
else:
self._berrors = unicode_errors
self.unicode_errors = PyBytes_AsString(self._berrors)
def __dealloc__(self):
free(self.pk.buf);
cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1:
cdef long long llval
cdef unsigned long long ullval
cdef long longval
cdef double fval
cdef char* rawval
cdef int ret
cdef dict d
if nest_limit < 0:
raise ValueError("Too deep.")
if o is None:
ret = msgpack_pack_nil(&self.pk)
elif isinstance(o, bool):
if o:
ret = msgpack_pack_true(&self.pk)
else:
ret = msgpack_pack_false(&self.pk)
elif PyLong_Check(o):
if o > 0:
ullval = o
ret = msgpack_pack_unsigned_long_long(&self.pk, ullval)
else:
llval = o
ret = msgpack_pack_long_long(&self.pk, llval)
elif PyInt_Check(o):
longval = o
ret = msgpack_pack_long(&self.pk, longval)
elif PyFloat_Check(o):
fval = o
ret = msgpack_pack_double(&self.pk, fval)
elif PyBytes_Check(o):
rawval = o
ret = msgpack_pack_raw(&self.pk, len(o))
if ret == 0:
ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
elif PyUnicode_Check(o):
if not self.encoding:
raise TypeError("Can't encode utf-8 no encoding is specified")
o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
rawval = o
ret = msgpack_pack_raw(&self.pk, len(o))
if ret == 0:
ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
elif PyDict_Check(o):
d = o
ret = msgpack_pack_map(&self.pk, len(d))
if ret == 0:
for k,v in d.items():
ret = self._pack(k, nest_limit-1)
if ret != 0: break
ret = self._pack(v, nest_limit-1)
if ret != 0: break
elif PySequence_Check(o):
ret = msgpack_pack_array(&self.pk, len(o))
if ret == 0:
for v in o:
ret = self._pack(v, nest_limit-1)
if ret != 0: break
elif self._default:
o = self._default(o)
ret = self._pack(o, nest_limit-1)
else:
raise TypeError("can't serialize %r" % (o,))
return ret
def pack(self, object obj):
cdef int ret
ret = self._pack(obj, DEFAULT_RECURSE_LIMIT)
if ret:
raise TypeError
buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
self.pk.length = 0
return buf
def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict'):
"""pack an object `o` and write it to stream)."""
packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors)
stream.write(packer.pack(o))
def packb(object o, default=None, encoding='utf-8', unicode_errors='strict'):
"""pack o and return packed bytes."""
packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors)
return packer.pack(o)
dumps = packs = packb
cdef extern from "unpack.h":
ctypedef struct msgpack_user:
int use_list
PyObject* object_hook
PyObject* list_hook
char *encoding
char *unicode_errors
ctypedef struct template_context:
msgpack_user user
PyObject* obj
size_t count
unsigned int ct
PyObject* key
int template_execute(template_context* ctx, const_char_ptr data,
size_t len, size_t* off) except -1
void template_init(template_context* ctx)
object template_data(template_context* ctx)
def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict"):
"""Unpack packed_bytes to object. Returns an unpacked object."""
cdef template_context ctx
cdef size_t off = 0
cdef int ret
cdef char* buf
cdef Py_ssize_t buf_len
PyObject_AsReadBuffer(packed, <const_void_ptr*>&buf, &buf_len)
if encoding is None:
enc = NULL
err = NULL
else:
if isinstance(encoding, unicode):
bencoding = encoding.encode('ascii')
else:
bencoding = encoding
if isinstance(unicode_errors, unicode):
berrors = unicode_errors.encode('ascii')
else:
berrors = unicode_errors
enc = PyBytes_AsString(bencoding)
err = PyBytes_AsString(berrors)
template_init(&ctx)
ctx.user.use_list = use_list
ctx.user.object_hook = ctx.user.list_hook = NULL
ctx.user.encoding = enc
ctx.user.unicode_errors = err
if object_hook is not None:
if not PyCallable_Check(object_hook):
raise TypeError("object_hook must be a callable.")
ctx.user.object_hook = <PyObject*>object_hook
if list_hook is not None:
if not PyCallable_Check(list_hook):
raise TypeError("list_hook must be a callable.")
ctx.user.list_hook = <PyObject*>list_hook
_gc_disable()
try:
ret = template_execute(&ctx, buf, buf_len, &off)
finally:
_gc_enable()
if ret == 1:
return template_data(&ctx)
else:
return None
loads = unpacks = unpackb
def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict"):
"""unpack an object from stream."""
return unpackb(stream.read(), use_list=use_list,
object_hook=object_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors)
cdef class Unpacker(object):
"""Unpacker(read_size=1024*1024)
Streaming unpacker.
read_size is used like file_like.read(read_size)
example:
unpacker = Unpacker()
while 1:
buf = astream.read()
unpacker.feed(buf)
for o in unpacker:
do_something(o)
"""
cdef template_context ctx
cdef char* buf
cdef size_t buf_size, buf_head, buf_tail
cdef object file_like
cdef object file_like_read
cdef Py_ssize_t read_size
cdef bint use_list
cdef object object_hook
cdef object _bencoding
cdef object _berrors
cdef char *encoding
cdef char *unicode_errors
def __cinit__(self):
self.buf = NULL
def __dealloc__(self):
free(self.buf);
self.buf = NULL;
def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0,
object object_hook=None, object list_hook=None,
encoding=None, unicode_errors='strict'):
if read_size == 0:
read_size = 1024*1024
self.use_list = use_list
self.file_like = file_like
if file_like:
self.file_like_read = file_like.read
if not PyCallable_Check(self.file_like_read):
raise ValueError("`file_like.read` must be a callable.")
self.read_size = read_size
self.buf = <char*>malloc(read_size)
if self.buf == NULL:
raise MemoryError("Unable to allocate internal buffer.")
self.buf_size = read_size
self.buf_head = 0
self.buf_tail = 0
template_init(&self.ctx)
self.ctx.user.use_list = use_list
self.ctx.user.object_hook = self.ctx.user.list_hook = <PyObject*>NULL
if object_hook is not None:
if not PyCallable_Check(object_hook):
raise TypeError("object_hook must be a callable.")
self.ctx.user.object_hook = <PyObject*>object_hook
if list_hook is not None:
if not PyCallable_Check(list_hook):
raise TypeError("list_hook must be a callable.")
self.ctx.user.list_hook = <PyObject*>list_hook
if encoding is None:
self.ctx.user.encoding = NULL
self.ctx.user.unicode_errors = NULL
else:
if isinstance(encoding, unicode):
self._bencoding = encoding.encode('ascii')
else:
self._bencoding = encoding
self.ctx.user.encoding = PyBytes_AsString(self._bencoding)
if isinstance(unicode_errors, unicode):
self._berrors = unicode_errors.encode('ascii')
else:
self._berrors = unicode_errors
self.ctx.user.unicode_errors = PyBytes_AsString(self._berrors)
def feed(self, object next_bytes):
cdef char* buf
cdef Py_ssize_t buf_len
if self.file_like is not None:
raise AssertionError(
"unpacker.feed() is not be able to use with`file_like`.")
PyObject_AsReadBuffer(next_bytes, <const_void_ptr*>&buf, &buf_len)
self.append_buffer(buf, buf_len)
cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len):
cdef:
char* buf = self.buf
size_t head = self.buf_head
size_t tail = self.buf_tail
size_t buf_size = self.buf_size
size_t new_size
if tail + _buf_len > buf_size:
if ((tail - head) + _buf_len)*2 < buf_size:
# move to front.
memmove(buf, buf + head, tail - head)
tail -= head
head = 0
else:
# expand buffer.
new_size = tail + _buf_len
if new_size < buf_size*2:
new_size = buf_size*2
buf = <char*>realloc(buf, new_size)
if buf == NULL:
# self.buf still holds old buffer and will be freed during
# obj destruction
raise MemoryError("Unable to enlarge internal buffer.")
buf_size = new_size
memcpy(buf + tail, <char*>(_buf), _buf_len)
self.buf = buf
self.buf_head = head
self.buf_size = buf_size
self.buf_tail = tail + _buf_len
# prepare self.buf from file_like
cdef fill_buffer(self):
if self.file_like is not None:
next_bytes = self.file_like_read(self.read_size)
if next_bytes:
self.append_buffer(PyBytes_AsString(next_bytes),
PyBytes_Size(next_bytes))
else:
self.file_like = None
cpdef unpack(self):
"""unpack one object"""
cdef int ret
while 1:
_gc_disable()
ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
_gc_enable()
if ret == 1:
o = template_data(&self.ctx)
template_init(&self.ctx)
return o
elif ret == 0:
if self.file_like is not None:
self.fill_buffer()
continue
raise StopIteration("No more unpack data.")
else:
raise ValueError("Unpack failed: error = %d" % (ret,))
def __iter__(self):
return self
def __next__(self):
return self.unpack()
# for debug.
#def _buf(self):
# return PyString_FromStringAndSize(self.buf, self.buf_tail)
#def _off(self):
# return self.buf_head

103
salt/msgpack/pack.h Normal file
View File

@ -0,0 +1,103 @@
/*
* MessagePack for Python packing routine
*
* Copyright (C) 2009 Naoki INADA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stddef.h>
#include <stdlib.h>
#include "sysdep.h"
#include "pack_define.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct msgpack_packer {
char *buf;
size_t length;
size_t buf_size;
} msgpack_packer;
typedef struct Packer Packer;
static inline int msgpack_pack_short(msgpack_packer* pk, short d);
static inline int msgpack_pack_int(msgpack_packer* pk, int d);
static inline int msgpack_pack_long(msgpack_packer* pk, long d);
static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d);
static inline int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d);
static inline int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d);
static inline int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d);
static inline int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d);
static inline int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d);
static inline int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d);
static inline int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d);
static inline int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d);
static inline int msgpack_pack_int8(msgpack_packer* pk, int8_t d);
static inline int msgpack_pack_int16(msgpack_packer* pk, int16_t d);
static inline int msgpack_pack_int32(msgpack_packer* pk, int32_t d);
static inline int msgpack_pack_int64(msgpack_packer* pk, int64_t d);
static inline int msgpack_pack_float(msgpack_packer* pk, float d);
static inline int msgpack_pack_double(msgpack_packer* pk, double d);
static inline int msgpack_pack_nil(msgpack_packer* pk);
static inline int msgpack_pack_true(msgpack_packer* pk);
static inline int msgpack_pack_false(msgpack_packer* pk);
static inline int msgpack_pack_array(msgpack_packer* pk, unsigned int n);
static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n);
static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l);
static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l);
static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l)
{
char* buf = pk->buf;
size_t bs = pk->buf_size;
size_t len = pk->length;
if (len + l > bs) {
bs = (len + l) * 2;
buf = realloc(buf, bs);
if (!buf) return -1;
}
memcpy(buf + len, data, l);
len += l;
pk->buf = buf;
pk->buf_size = bs;
pk->length = len;
return 0;
}
#define msgpack_pack_inline_func(name) \
static inline int msgpack_pack ## name
#define msgpack_pack_inline_func_cint(name) \
static inline int msgpack_pack ## name
#define msgpack_pack_user msgpack_packer*
#define msgpack_pack_append_buffer(user, buf, len) \
return msgpack_pack_write(user, (const char*)buf, len)
#include "pack_template.h"
#ifdef __cplusplus
}
#endif

25
salt/msgpack/pack_define.h Normal file
View File

@ -0,0 +1,25 @@
/*
* MessagePack unpacking routine template
*
* Copyright (C) 2008-2009 FURUHASHI Sadayuki
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MSGPACK_PACK_DEFINE_H__
#define MSGPACK_PACK_DEFINE_H__
#include "sysdep.h"
#include <limits.h>
#endif /* msgpack/pack_define.h */

686
salt/msgpack/pack_template.h Normal file
View File

@ -0,0 +1,686 @@
/*
* MessagePack packing routine template
*
* Copyright (C) 2008-2009 FURUHASHI Sadayuki
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifdef __LITTLE_ENDIAN__
#define TAKE8_8(d) ((uint8_t*)&d)[0]
#define TAKE8_16(d) ((uint8_t*)&d)[0]
#define TAKE8_32(d) ((uint8_t*)&d)[0]
#define TAKE8_64(d) ((uint8_t*)&d)[0]
#elif __BIG_ENDIAN__
#define TAKE8_8(d) ((uint8_t*)&d)[0]
#define TAKE8_16(d) ((uint8_t*)&d)[1]
#define TAKE8_32(d) ((uint8_t*)&d)[3]
#define TAKE8_64(d) ((uint8_t*)&d)[7]
#endif
#ifndef msgpack_pack_inline_func
#error msgpack_pack_inline_func template is not defined
#endif
#ifndef msgpack_pack_user
#error msgpack_pack_user type is not defined
#endif
#ifndef msgpack_pack_append_buffer
#error msgpack_pack_append_buffer callback is not defined
#endif
/*
* Integer
*/
#define msgpack_pack_real_uint8(x, d) \
do { \
if(d < (1<<7)) { \
/* fixnum */ \
msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \
} else { \
/* unsigned 8 */ \
unsigned char buf[2] = {0xcc, TAKE8_8(d)}; \
msgpack_pack_append_buffer(x, buf, 2); \
} \
} while(0)
#define msgpack_pack_real_uint16(x, d) \
do { \
if(d < (1<<7)) { \
/* fixnum */ \
msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \
} else if(d < (1<<8)) { \
/* unsigned 8 */ \
unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \
msgpack_pack_append_buffer(x, buf, 2); \
} else { \
/* unsigned 16 */ \
unsigned char buf[3]; \
buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \
msgpack_pack_append_buffer(x, buf, 3); \
} \
} while(0)
#define msgpack_pack_real_uint32(x, d) \
do { \
if(d < (1<<8)) { \
if(d < (1<<7)) { \
/* fixnum */ \
msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \
} else { \
/* unsigned 8 */ \
unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \
msgpack_pack_append_buffer(x, buf, 2); \
} \
} else { \
if(d < (1<<16)) { \
/* unsigned 16 */ \
unsigned char buf[3]; \
buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \
msgpack_pack_append_buffer(x, buf, 3); \
} else { \
/* unsigned 32 */ \
unsigned char buf[5]; \
buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \
msgpack_pack_append_buffer(x, buf, 5); \
} \
} \
} while(0)
#define msgpack_pack_real_uint64(x, d) \
do { \
if(d < (1ULL<<8)) { \
if(d < (1<<7)) { \
/* fixnum */ \
msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \
} else { \
/* unsigned 8 */ \
unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \
msgpack_pack_append_buffer(x, buf, 2); \
} \
} else { \
if(d < (1ULL<<16)) { \
/* signed 16 */ \
unsigned char buf[3]; \
buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \
msgpack_pack_append_buffer(x, buf, 3); \
} else if(d < (1ULL<<32)) { \
/* signed 32 */ \
unsigned char buf[5]; \
buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \
msgpack_pack_append_buffer(x, buf, 5); \
} else { \
/* signed 64 */ \
unsigned char buf[9]; \
buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \
msgpack_pack_append_buffer(x, buf, 9); \
} \
} \
} while(0)
#define msgpack_pack_real_int8(x, d) \
do { \
if(d < -(1<<5)) { \
/* signed 8 */ \
unsigned char buf[2] = {0xd0, TAKE8_8(d)}; \
msgpack_pack_append_buffer(x, buf, 2); \
} else { \
/* fixnum */ \
msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \
} \
} while(0)
#define msgpack_pack_real_int16(x, d) \
do { \
if(d < -(1<<5)) { \
if(d < -(1<<7)) { \
/* signed 16 */ \
unsigned char buf[3]; \
buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \
msgpack_pack_append_buffer(x, buf, 3); \
} else { \
/* signed 8 */ \
unsigned char buf[2] = {0xd0, TAKE8_16(d)}; \
msgpack_pack_append_buffer(x, buf, 2); \
} \
} else if(d < (1<<7)) { \
/* fixnum */ \
msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \
} else { \
if(d < (1<<8)) { \
/* unsigned 8 */ \
unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \
msgpack_pack_append_buffer(x, buf, 2); \
} else { \
/* unsigned 16 */ \
unsigned char buf[3]; \
buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \
msgpack_pack_append_buffer(x, buf, 3); \
} \
} \
} while(0)
#define msgpack_pack_real_int32(x, d) \
do { \
if(d < -(1<<5)) { \
if(d < -(1<<15)) { \
/* signed 32 */ \
unsigned char buf[5]; \
buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \
msgpack_pack_append_buffer(x, buf, 5); \
} else if(d < -(1<<7)) { \
/* signed 16 */ \
unsigned char buf[3]; \
buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \
msgpack_pack_append_buffer(x, buf, 3); \
} else { \
/* signed 8 */ \
unsigned char buf[2] = {0xd0, TAKE8_32(d)}; \
msgpack_pack_append_buffer(x, buf, 2); \
} \
} else if(d < (1<<7)) { \
/* fixnum */ \
msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \
} else { \
if(d < (1<<8)) { \
/* unsigned 8 */ \
unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \
msgpack_pack_append_buffer(x, buf, 2); \
} else if(d < (1<<16)) { \
/* unsigned 16 */ \
unsigned char buf[3]; \
buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \
msgpack_pack_append_buffer(x, buf, 3); \
} else { \
/* unsigned 32 */ \
unsigned char buf[5]; \
buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \
msgpack_pack_append_buffer(x, buf, 5); \
} \
} \
} while(0)
#define msgpack_pack_real_int64(x, d) \
do { \
if(d < -(1LL<<5)) { \
if(d < -(1LL<<15)) { \
if(d < -(1LL<<31)) { \
/* signed 64 */ \
unsigned char buf[9]; \
buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); \
msgpack_pack_append_buffer(x, buf, 9); \
} else { \
/* signed 32 */ \
unsigned char buf[5]; \
buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \
msgpack_pack_append_buffer(x, buf, 5); \
} \
} else { \
if(d < -(1<<7)) { \
/* signed 16 */ \
unsigned char buf[3]; \
buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \
msgpack_pack_append_buffer(x, buf, 3); \
} else { \
/* signed 8 */ \
unsigned char buf[2] = {0xd0, TAKE8_64(d)}; \
msgpack_pack_append_buffer(x, buf, 2); \
} \
} \
} else if(d < (1<<7)) { \
/* fixnum */ \
msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \
} else { \
if(d < (1LL<<16)) { \
if(d < (1<<8)) { \
/* unsigned 8 */ \
unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \
msgpack_pack_append_buffer(x, buf, 2); \
} else { \
/* unsigned 16 */ \
unsigned char buf[3]; \
buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \
msgpack_pack_append_buffer(x, buf, 3); \
} \
} else { \
if(d < (1LL<<32)) { \
/* unsigned 32 */ \
unsigned char buf[5]; \
buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \
msgpack_pack_append_buffer(x, buf, 5); \
} else { \
/* unsigned 64 */ \
unsigned char buf[9]; \
buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \
msgpack_pack_append_buffer(x, buf, 9); \
} \
} \
} \
} while(0)
#ifdef msgpack_pack_inline_func_fastint
msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d)
{
unsigned char buf[2] = {0xcc, TAKE8_8(d)};
msgpack_pack_append_buffer(x, buf, 2);
}
msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d)
{
unsigned char buf[3];
buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d);
msgpack_pack_append_buffer(x, buf, 3);
}
msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d)
{
unsigned char buf[5];
buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d);
msgpack_pack_append_buffer(x, buf, 5);
}
msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d)
{
unsigned char buf[9];
buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d);
msgpack_pack_append_buffer(x, buf, 9);
}
msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d)
{
unsigned char buf[2] = {0xd0, TAKE8_8(d)};
msgpack_pack_append_buffer(x, buf, 2);
}
msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d)
{
unsigned char buf[3];
buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d);
msgpack_pack_append_buffer(x, buf, 3);
}
msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d)
{
unsigned char buf[5];
buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d);
msgpack_pack_append_buffer(x, buf, 5);
}
msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d)
{
unsigned char buf[9];
buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d);
msgpack_pack_append_buffer(x, buf, 9);
}
#undef msgpack_pack_inline_func_fastint
#endif
msgpack_pack_inline_func(_uint8)(msgpack_pack_user x, uint8_t d)
{
msgpack_pack_real_uint8(x, d);
}
msgpack_pack_inline_func(_uint16)(msgpack_pack_user x, uint16_t d)
{
msgpack_pack_real_uint16(x, d);
}
msgpack_pack_inline_func(_uint32)(msgpack_pack_user x, uint32_t d)
{
msgpack_pack_real_uint32(x, d);
}
msgpack_pack_inline_func(_uint64)(msgpack_pack_user x, uint64_t d)
{
msgpack_pack_real_uint64(x, d);
}
msgpack_pack_inline_func(_int8)(msgpack_pack_user x, int8_t d)
{
msgpack_pack_real_int8(x, d);
}
msgpack_pack_inline_func(_int16)(msgpack_pack_user x, int16_t d)
{
msgpack_pack_real_int16(x, d);
}
msgpack_pack_inline_func(_int32)(msgpack_pack_user x, int32_t d)
{
msgpack_pack_real_int32(x, d);
}
msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d)
{
msgpack_pack_real_int64(x, d);
}
#ifdef msgpack_pack_inline_func_cint
msgpack_pack_inline_func_cint(_short)(msgpack_pack_user x, short d)
{
#if defined(SIZEOF_SHORT) || defined(SHRT_MAX)
#if SIZEOF_SHORT == 2 || SHRT_MAX == 0x7fff
msgpack_pack_real_int16(x, d);
#elif SIZEOF_SHORT == 4 || SHRT_MAX == 0x7fffffff
msgpack_pack_real_int32(x, d);
#else
msgpack_pack_real_int64(x, d);
#endif
#else
if(sizeof(short) == 2) {
msgpack_pack_real_int16(x, d);
} else if(sizeof(short) == 4) {
msgpack_pack_real_int32(x, d);
} else {
msgpack_pack_real_int64(x, d);
}
#endif
}
msgpack_pack_inline_func_cint(_int)(msgpack_pack_user x, int d)
{
#if defined(SIZEOF_INT) || defined(INT_MAX)
#if SIZEOF_INT == 2 || INT_MAX == 0x7fff
msgpack_pack_real_int16(x, d);
#elif SIZEOF_INT == 4 || INT_MAX == 0x7fffffff
msgpack_pack_real_int32(x, d);
#else
msgpack_pack_real_int64(x, d);
#endif
#else
if(sizeof(int) == 2) {
msgpack_pack_real_int16(x, d);
} else if(sizeof(int) == 4) {
msgpack_pack_real_int32(x, d);
} else {
msgpack_pack_real_int64(x, d);
}
#endif
}
msgpack_pack_inline_func_cint(_long)(msgpack_pack_user x, long d)
{
#if defined(SIZEOF_LONG) || defined(LONG_MAX)
#if SIZEOF_LONG == 2 || LONG_MAX == 0x7fffL
msgpack_pack_real_int16(x, d);
#elif SIZEOF_LONG == 4 || LONG_MAX == 0x7fffffffL
msgpack_pack_real_int32(x, d);
#else
msgpack_pack_real_int64(x, d);
#endif
#else
if(sizeof(long) == 2) {
msgpack_pack_real_int16(x, d);
} else if(sizeof(long) == 4) {
msgpack_pack_real_int32(x, d);
} else {
msgpack_pack_real_int64(x, d);
}
#endif
}
msgpack_pack_inline_func_cint(_long_long)(msgpack_pack_user x, long long d)
{
#if defined(SIZEOF_LONG_LONG) || defined(LLONG_MAX)
#if SIZEOF_LONG_LONG == 2 || LLONG_MAX == 0x7fffL
msgpack_pack_real_int16(x, d);
#elif SIZEOF_LONG_LONG == 4 || LLONG_MAX == 0x7fffffffL
msgpack_pack_real_int32(x, d);
#else
msgpack_pack_real_int64(x, d);
#endif
#else
if(sizeof(long long) == 2) {
msgpack_pack_real_int16(x, d);
} else if(sizeof(long long) == 4) {
msgpack_pack_real_int32(x, d);
} else {
msgpack_pack_real_int64(x, d);
}
#endif
}
msgpack_pack_inline_func_cint(_unsigned_short)(msgpack_pack_user x, unsigned short d)
{
#if defined(SIZEOF_SHORT) || defined(USHRT_MAX)
#if SIZEOF_SHORT == 2 || USHRT_MAX == 0xffffU
msgpack_pack_real_uint16(x, d);
#elif SIZEOF_SHORT == 4 || USHRT_MAX == 0xffffffffU
msgpack_pack_real_uint32(x, d);
#else
msgpack_pack_real_uint64(x, d);
#endif
#else
if(sizeof(unsigned short) == 2) {
msgpack_pack_real_uint16(x, d);
} else if(sizeof(unsigned short) == 4) {
msgpack_pack_real_uint32(x, d);
} else {
msgpack_pack_real_uint64(x, d);
}
#endif
}
msgpack_pack_inline_func_cint(_unsigned_int)(msgpack_pack_user x, unsigned int d)
{
#if defined(SIZEOF_INT) || defined(UINT_MAX)
#if SIZEOF_INT == 2 || UINT_MAX == 0xffffU
msgpack_pack_real_uint16(x, d);
#elif SIZEOF_INT == 4 || UINT_MAX == 0xffffffffU
msgpack_pack_real_uint32(x, d);
#else
msgpack_pack_real_uint64(x, d);
#endif
#else
if(sizeof(unsigned int) == 2) {
msgpack_pack_real_uint16(x, d);
} else if(sizeof(unsigned int) == 4) {
msgpack_pack_real_uint32(x, d);
} else {
msgpack_pack_real_uint64(x, d);
}
#endif
}
msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long d)
{
#if defined(SIZEOF_LONG) || defined(ULONG_MAX)
#if SIZEOF_LONG == 2 || ULONG_MAX == 0xffffUL
msgpack_pack_real_uint16(x, d);
#elif SIZEOF_LONG == 4 || ULONG_MAX == 0xffffffffUL
msgpack_pack_real_uint32(x, d);
#else
msgpack_pack_real_uint64(x, d);
#endif
#else
if(sizeof(unsigned int) == 2) {
msgpack_pack_real_uint16(x, d);
} else if(sizeof(unsigned int) == 4) {
msgpack_pack_real_uint32(x, d);
} else {
msgpack_pack_real_uint64(x, d);
}
#endif
}
msgpack_pack_inline_func_cint(_unsigned_long_long)(msgpack_pack_user x, unsigned long long d)
{
#if defined(SIZEOF_LONG_LONG) || defined(ULLONG_MAX)
#if SIZEOF_LONG_LONG == 2 || ULLONG_MAX == 0xffffUL
msgpack_pack_real_uint16(x, d);
#elif SIZEOF_LONG_LONG == 4 || ULLONG_MAX == 0xffffffffUL
msgpack_pack_real_uint32(x, d);
#else
msgpack_pack_real_uint64(x, d);
#endif
#else
if(sizeof(unsigned long long) == 2) {
msgpack_pack_real_uint16(x, d);
} else if(sizeof(unsigned long long) == 4) {
msgpack_pack_real_uint32(x, d);
} else {
msgpack_pack_real_uint64(x, d);
}
#endif
}
#undef msgpack_pack_inline_func_cint
#endif
/*
* Float
*/
msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d)
{
union { char buf[4]; uint32_t num; } f;
*((float*)&f.buf) = d; // FIXME
unsigned char buf[5];
buf[0] = 0xca; *(uint32_t*)&buf[1] = _msgpack_be32(f.num);
msgpack_pack_append_buffer(x, buf, 5);
}
msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d)
{
union { char buf[8]; uint64_t num; } f;
*((double*)&f.buf) = d; // FIXME
unsigned char buf[9];
buf[0] = 0xcb; *(uint64_t*)&buf[1] = _msgpack_be64(f.num);
msgpack_pack_append_buffer(x, buf, 9);
}
/*
* Nil
*/
msgpack_pack_inline_func(_nil)(msgpack_pack_user x)
{
static const unsigned char d = 0xc0;
msgpack_pack_append_buffer(x, &d, 1);
}
/*
* Boolean
*/
msgpack_pack_inline_func(_true)(msgpack_pack_user x)
{
static const unsigned char d = 0xc3;
msgpack_pack_append_buffer(x, &d, 1);
}
msgpack_pack_inline_func(_false)(msgpack_pack_user x)
{
static const unsigned char d = 0xc2;
msgpack_pack_append_buffer(x, &d, 1);
}
/*
* Array
*/
msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n)
{
if(n < 16) {
unsigned char d = 0x90 | n;
msgpack_pack_append_buffer(x, &d, 1);
} else if(n < 65536) {
unsigned char buf[3];
buf[0] = 0xdc; *(uint16_t*)&buf[1] = _msgpack_be16(n);
msgpack_pack_append_buffer(x, buf, 3);
} else {
unsigned char buf[5];
buf[0] = 0xdd; *(uint32_t*)&buf[1] = _msgpack_be32(n);
msgpack_pack_append_buffer(x, buf, 5);
}
}
/*
* Map
*/
msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n)
{
if(n < 16) {
unsigned char d = 0x80 | n;
msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
} else if(n < 65536) {
unsigned char buf[3];
buf[0] = 0xde; *(uint16_t*)&buf[1] = _msgpack_be16(n);
msgpack_pack_append_buffer(x, buf, 3);
} else {
unsigned char buf[5];
buf[0] = 0xdf; *(uint32_t*)&buf[1] = _msgpack_be32(n);
msgpack_pack_append_buffer(x, buf, 5);
}
}
/*
* Raw
*/
msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l)
{
if(l < 32) {
unsigned char d = 0xa0 | l;
msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
} else if(l < 65536) {
unsigned char buf[3];
buf[0] = 0xda; *(uint16_t*)&buf[1] = _msgpack_be16(l);
msgpack_pack_append_buffer(x, buf, 3);
} else {
unsigned char buf[5];
buf[0] = 0xdb; *(uint32_t*)&buf[1] = _msgpack_be32(l);
msgpack_pack_append_buffer(x, buf, 5);
}
}
msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l)
{
msgpack_pack_append_buffer(x, (const unsigned char*)b, l);
}
#undef msgpack_pack_inline_func
#undef msgpack_pack_user
#undef msgpack_pack_append_buffer
#undef TAKE8_8
#undef TAKE8_16
#undef TAKE8_32
#undef TAKE8_64
#undef msgpack_pack_real_uint8
#undef msgpack_pack_real_uint16
#undef msgpack_pack_real_uint32
#undef msgpack_pack_real_uint64
#undef msgpack_pack_real_int8
#undef msgpack_pack_real_int16
#undef msgpack_pack_real_int32
#undef msgpack_pack_real_int64

94
salt/msgpack/sysdep.h Normal file
View File

@ -0,0 +1,94 @@
/*
* MessagePack system dependencies
*
* Copyright (C) 2008-2009 FURUHASHI Sadayuki
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MSGPACK_SYSDEP_H__
#define MSGPACK_SYSDEP_H__
#ifdef _MSC_VER
typedef __int8 int8_t;
typedef unsigned __int8 uint8_t;
typedef __int16 int16_t;
typedef unsigned __int16 uint16_t;
typedef __int32 int32_t;
typedef unsigned __int32 uint32_t;
typedef __int64 int64_t;
typedef unsigned __int64 uint64_t;
#else
#include <stddef.h>
#include <stdint.h>
#include <stdbool.h>
#endif
#ifdef _WIN32
typedef long _msgpack_atomic_counter_t;
#define _msgpack_sync_decr_and_fetch(ptr) InterlockedDecrement(ptr)
#define _msgpack_sync_incr_and_fetch(ptr) InterlockedIncrement(ptr)
#else
typedef unsigned int _msgpack_atomic_counter_t;
#define _msgpack_sync_decr_and_fetch(ptr) __sync_sub_and_fetch(ptr, 1)
#define _msgpack_sync_incr_and_fetch(ptr) __sync_add_and_fetch(ptr, 1)
#endif
#ifdef _WIN32
#include <winsock2.h>
#else
#include <arpa/inet.h> /* __BYTE_ORDER */
#endif
#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
#if __BYTE_ORDER == __LITTLE_ENDIAN
#define __LITTLE_ENDIAN__
#elif __BYTE_ORDER == __BIG_ENDIAN
#define __BIG_ENDIAN__
#endif
#endif
#ifdef __LITTLE_ENDIAN__
#define _msgpack_be16(x) ntohs(x)
#define _msgpack_be32(x) ntohl(x)
#if defined(_byteswap_uint64)
# define _msgpack_be64(x) (_byteswap_uint64(x))
#elif defined(bswap_64)
# define _msgpack_be64(x) bswap_64(x)
#elif defined(__DARWIN_OSSwapInt64)
# define _msgpack_be64(x) __DARWIN_OSSwapInt64(x)
#else
#define _msgpack_be64(x) \
( ((((uint64_t)x) << 56) & 0xff00000000000000ULL ) | \
((((uint64_t)x) << 40) & 0x00ff000000000000ULL ) | \
((((uint64_t)x) << 24) & 0x0000ff0000000000ULL ) | \
((((uint64_t)x) << 8) & 0x000000ff00000000ULL ) | \
((((uint64_t)x) >> 8) & 0x00000000ff000000ULL ) | \
((((uint64_t)x) >> 24) & 0x0000000000ff0000ULL ) | \
((((uint64_t)x) >> 40) & 0x000000000000ff00ULL ) | \
((((uint64_t)x) >> 56) & 0x00000000000000ffULL ) )
#endif
#else
#define _msgpack_be16(x) (x)
#define _msgpack_be32(x) (x)
#define _msgpack_be64(x) (x)
#endif
#endif /* msgpack/sysdep.h */

213
salt/msgpack/unpack.h Normal file
View File

@ -0,0 +1,213 @@
/*
* MessagePack for Python unpacking routine
*
* Copyright (C) 2009 Naoki INADA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define MSGPACK_MAX_STACK_SIZE (1024)
#include "unpack_define.h"
typedef struct unpack_user {
int use_list;
PyObject *object_hook;
PyObject *list_hook;
const char *encoding;
const char *unicode_errors;
} unpack_user;
#define msgpack_unpack_struct(name) \
struct template ## name
#define msgpack_unpack_func(ret, name) \
static inline ret template ## name
#define msgpack_unpack_callback(name) \
template_callback ## name
#define msgpack_unpack_object PyObject*
#define msgpack_unpack_user unpack_user
struct template_context;
typedef struct template_context template_context;
static inline msgpack_unpack_object template_callback_root(unpack_user* u)
{
return NULL;
}
static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o)
{
PyObject *p = PyInt_FromLong((long)d);
if (!p)
return -1;
*o = p;
return 0;
}
static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o)
{
return template_callback_uint16(u, d, o);
}
static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o)
{
PyObject *p;
if (d > LONG_MAX) {
p = PyLong_FromUnsignedLong((unsigned long)d);
} else {
p = PyInt_FromLong((long)d);
}
if (!p)
return -1;
*o = p;
return 0;
}
static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o)
{
PyObject *p = PyLong_FromUnsignedLongLong(d);
if (!p)
return -1;
*o = p;
return 0;
}
static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o)
{
PyObject *p = PyInt_FromLong(d);
if (!p)
return -1;
*o = p;
return 0;
}
static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o)
{
return template_callback_int32(u, d, o);
}
static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o)
{
return template_callback_int32(u, d, o);
}
static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o)
{
PyObject *p = PyLong_FromLongLong(d);
if (!p)
return -1;
*o = p;
return 0;
}
static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o)
{
PyObject *p = PyFloat_FromDouble(d);
if (!p)
return -1;
*o = p;
return 0;
}
static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o)
{
return template_callback_double(u, d, o);
}
static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o)
{ Py_INCREF(Py_None); *o = Py_None; return 0; }
static inline int template_callback_true(unpack_user* u, msgpack_unpack_object* o)
{ Py_INCREF(Py_True); *o = Py_True; return 0; }
static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* o)
{ Py_INCREF(Py_False); *o = Py_False; return 0; }
static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
{
PyObject *p = u->use_list ? PyList_New(n) : PyTuple_New(n);
if (!p)
return -1;
*o = p;
return 0;
}
static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o)
{
if (u->use_list)
PyList_SET_ITEM(*c, current, o);
else
PyTuple_SET_ITEM(*c, current, o);
return 0;
}
static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_object* c)
{
if (u->list_hook) {
PyObject *arglist = Py_BuildValue("(O)", *c);
*c = PyEval_CallObject(u->list_hook, arglist);
Py_DECREF(arglist);
}
return 0;
}
static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
{
PyObject *p = PyDict_New();
if (!p)
return -1;
*o = p;
return 0;
}
static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v)
{
if (PyDict_SetItem(*c, k, v) == 0) {
Py_DECREF(k);
Py_DECREF(v);
return 0;
}
return -1;
}
static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
{
if (u->object_hook) {
PyObject *arglist = Py_BuildValue("(O)", *c);
*c = PyEval_CallObject(u->object_hook, arglist);
Py_DECREF(arglist);
}
return 0;
}
static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
{
PyObject *py;
if(u->encoding) {
py = PyUnicode_Decode(p, l, u->encoding, u->unicode_errors);
} else {
py = PyBytes_FromStringAndSize(p, l);
}
if (!py)
return -1;
*o = py;
return 0;
}
#include "unpack_template.h"

92
salt/msgpack/unpack_define.h Normal file
View File

@ -0,0 +1,92 @@
/*
* MessagePack unpacking routine template
*
* Copyright (C) 2008-2009 FURUHASHI Sadayuki
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MSGPACK_UNPACK_DEFINE_H__
#define MSGPACK_UNPACK_DEFINE_H__
#include "sysdep.h"
#include <string.h>
#include <assert.h>
#include <stdio.h>
#ifdef __cplusplus
extern "C" {
#endif
#ifndef MSGPACK_MAX_STACK_SIZE
#define MSGPACK_MAX_STACK_SIZE 16
#endif
typedef enum {
CS_HEADER = 0x00, // nil
//CS_ = 0x01,
//CS_ = 0x02, // false
//CS_ = 0x03, // true
//CS_ = 0x04,
//CS_ = 0x05,
//CS_ = 0x06,
//CS_ = 0x07,
//CS_ = 0x08,
//CS_ = 0x09,
CS_FLOAT = 0x0a,
CS_DOUBLE = 0x0b,
CS_UINT_8 = 0x0c,
CS_UINT_16 = 0x0d,
CS_UINT_32 = 0x0e,
CS_UINT_64 = 0x0f,
CS_INT_8 = 0x10,
CS_INT_16 = 0x11,
CS_INT_32 = 0x12,
CS_INT_64 = 0x13,
//CS_ = 0x14,
//CS_ = 0x15,
//CS_BIG_INT_16 = 0x16,
//CS_BIG_INT_32 = 0x17,
//CS_BIG_FLOAT_16 = 0x18,
//CS_BIG_FLOAT_32 = 0x19,
CS_RAW_16 = 0x1a,
CS_RAW_32 = 0x1b,
CS_ARRAY_16 = 0x1c,
CS_ARRAY_32 = 0x1d,
CS_MAP_16 = 0x1e,
CS_MAP_32 = 0x1f,
//ACS_BIG_INT_VALUE,
//ACS_BIG_FLOAT_VALUE,
ACS_RAW_VALUE,
} msgpack_unpack_state;
typedef enum {
CT_ARRAY_ITEM,
CT_MAP_KEY,
CT_MAP_VALUE,
} msgpack_container_type;
#ifdef __cplusplus
}
#endif
#endif /* msgpack/unpack_define.h */

385
salt/msgpack/unpack_template.h Normal file
View File

@ -0,0 +1,385 @@
/*
* MessagePack unpacking routine template
*
* Copyright (C) 2008-2009 FURUHASHI Sadayuki
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef msgpack_unpack_func
#error msgpack_unpack_func template is not defined
#endif
#ifndef msgpack_unpack_callback
#error msgpack_unpack_callback template is not defined
#endif
#ifndef msgpack_unpack_struct
#error msgpack_unpack_struct template is not defined
#endif
#ifndef msgpack_unpack_struct_decl
#define msgpack_unpack_struct_decl(name) msgpack_unpack_struct(name)
#endif
#ifndef msgpack_unpack_object
#error msgpack_unpack_object type is not defined
#endif
#ifndef msgpack_unpack_user
#error msgpack_unpack_user type is not defined
#endif
#ifndef USE_CASE_RANGE
#if !defined(_MSC_VER)
#define USE_CASE_RANGE
#endif
#endif
msgpack_unpack_struct_decl(_stack) {
msgpack_unpack_object obj;
size_t count;
unsigned int ct;
union {
size_t curr;
msgpack_unpack_object map_key;
};
};
msgpack_unpack_struct_decl(_context) {
msgpack_unpack_user user;
unsigned int cs;
unsigned int trail;
unsigned int top;
msgpack_unpack_struct(_stack) stack[MSGPACK_MAX_STACK_SIZE];
};
msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx)
{
ctx->cs = CS_HEADER;
ctx->trail = 0;
ctx->top = 0;
ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user);
}
msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx)
{
return (ctx)->stack[0].obj;
}
msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off)
{
assert(len >= *off);
const unsigned char* p = (unsigned char*)data + *off;
const unsigned char* const pe = (unsigned char*)data + len;
const void* n = NULL;
unsigned int trail = ctx->trail;
unsigned int cs = ctx->cs;
unsigned int top = ctx->top;
msgpack_unpack_struct(_stack)* stack = ctx->stack;
msgpack_unpack_user* user = &ctx->user;
msgpack_unpack_object obj;
msgpack_unpack_struct(_stack)* c = NULL;
int ret;
#define push_simple_value(func) \
if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \
goto _push
#define push_fixed_value(func, arg) \
if(msgpack_unpack_callback(func)(user, arg, &obj) < 0) { goto _failed; } \
goto _push
#define push_variable_value(func, base, pos, len) \
if(msgpack_unpack_callback(func)(user, \
(const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \
goto _push
#define again_fixed_trail(_cs, trail_len) \
trail = trail_len; \
cs = _cs; \
goto _fixed_trail_again
#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \
trail = trail_len; \
if(trail == 0) { goto ifzero; } \
cs = _cs; \
goto _fixed_trail_again
#define start_container(func, count_, ct_) \
if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \
if((count_) == 0) { obj = stack[top].obj; goto _push; } \
if(top >= MSGPACK_MAX_STACK_SIZE) { goto _failed; } \
stack[top].ct = ct_; \
stack[top].curr = 0; \
stack[top].count = count_; \
/*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \
/*printf("stack push %d\n", top);*/ \
++top; \
goto _header_again
#define NEXT_CS(p) \
((unsigned int)*p & 0x1f)
#define PTR_CAST_8(ptr) (*(uint8_t*)ptr)
#define PTR_CAST_16(ptr) _msgpack_be16(*(uint16_t*)ptr)
#define PTR_CAST_32(ptr) _msgpack_be32(*(uint32_t*)ptr)
#define PTR_CAST_64(ptr) _msgpack_be64(*(uint64_t*)ptr)
#ifdef USE_CASE_RANGE
#define SWITCH_RANGE_BEGIN switch(*p) {
#define SWITCH_RANGE(FROM, TO) case FROM ... TO:
#define SWITCH_RANGE_DEFAULT default:
#define SWITCH_RANGE_END }
#else
#define SWITCH_RANGE_BEGIN { if(0) {
#define SWITCH_RANGE(FROM, TO) } else if(FROM <= *p && *p <= TO) {
#define SWITCH_RANGE_DEFAULT } else {
#define SWITCH_RANGE_END } }
#endif
if(p == pe) { goto _out; }
do {
switch(cs) {
case CS_HEADER:
SWITCH_RANGE_BEGIN
SWITCH_RANGE(0x00, 0x7f) // Positive Fixnum
push_fixed_value(_uint8, *(uint8_t*)p);
SWITCH_RANGE(0xe0, 0xff) // Negative Fixnum
push_fixed_value(_int8, *(int8_t*)p);
SWITCH_RANGE(0xc0, 0xdf) // Variable
switch(*p) {
case 0xc0: // nil
push_simple_value(_nil);
//case 0xc1: // string
// again_terminal_trail(NEXT_CS(p), p+1);
case 0xc2: // false
push_simple_value(_false);
case 0xc3: // true
push_simple_value(_true);
//case 0xc4:
//case 0xc5:
//case 0xc6:
//case 0xc7:
//case 0xc8:
//case 0xc9:
case 0xca: // float
case 0xcb: // double
case 0xcc: // unsigned int 8
case 0xcd: // unsigned int 16
case 0xce: // unsigned int 32
case 0xcf: // unsigned int 64
case 0xd0: // signed int 8
case 0xd1: // signed int 16
case 0xd2: // signed int 32
case 0xd3: // signed int 64
again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03));
//case 0xd4:
//case 0xd5:
//case 0xd6: // big integer 16
//case 0xd7: // big integer 32
//case 0xd8: // big float 16
//case 0xd9: // big float 32
case 0xda: // raw 16
case 0xdb: // raw 32
case 0xdc: // array 16
case 0xdd: // array 32
case 0xde: // map 16
case 0xdf: // map 32
again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01));
default:
goto _failed;
}
SWITCH_RANGE(0xa0, 0xbf) // FixRaw
again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero);
SWITCH_RANGE(0x90, 0x9f) // FixArray
start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM);
SWITCH_RANGE(0x80, 0x8f) // FixMap
start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY);
SWITCH_RANGE_DEFAULT
goto _failed;
SWITCH_RANGE_END
// end CS_HEADER
_fixed_trail_again:
++p;
default:
if((size_t)(pe - p) < trail) { goto _out; }
n = p; p += trail - 1;
switch(cs) {
//case CS_
//case CS_
case CS_FLOAT: {
union { uint32_t num; char buf[4]; } f;
f.num = PTR_CAST_32(n); // FIXME
push_fixed_value(_float, *((float*)f.buf)); }
case CS_DOUBLE: {
union { uint64_t num; char buf[8]; } f;
f.num = PTR_CAST_64(n); // FIXME
push_fixed_value(_double, *((double*)f.buf)); }
case CS_UINT_8:
push_fixed_value(_uint8, (uint8_t)PTR_CAST_8(n));
case CS_UINT_16:
push_fixed_value(_uint16, (uint16_t)PTR_CAST_16(n));
case CS_UINT_32:
push_fixed_value(_uint32, (uint32_t)PTR_CAST_32(n));
case CS_UINT_64:
push_fixed_value(_uint64, (uint64_t)PTR_CAST_64(n));
case CS_INT_8:
push_fixed_value(_int8, (int8_t)PTR_CAST_8(n));
case CS_INT_16:
push_fixed_value(_int16, (int16_t)PTR_CAST_16(n));
case CS_INT_32:
push_fixed_value(_int32, (int32_t)PTR_CAST_32(n));
case CS_INT_64:
push_fixed_value(_int64, (int64_t)PTR_CAST_64(n));
//case CS_
//case CS_
//case CS_BIG_INT_16:
// again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint16_t)PTR_CAST_16(n), _big_int_zero);
//case CS_BIG_INT_32:
// again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint32_t)PTR_CAST_32(n), _big_int_zero);
//case ACS_BIG_INT_VALUE:
//_big_int_zero:
// // FIXME
// push_variable_value(_big_int, data, n, trail);
//case CS_BIG_FLOAT_16:
// again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero);
//case CS_BIG_FLOAT_32:
// again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint32_t)PTR_CAST_32(n), _big_float_zero);
//case ACS_BIG_FLOAT_VALUE:
//_big_float_zero:
// // FIXME
// push_variable_value(_big_float, data, n, trail);
case CS_RAW_16:
again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero);
case CS_RAW_32:
again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero);
case ACS_RAW_VALUE:
_raw_zero:
push_variable_value(_raw, data, n, trail);
case CS_ARRAY_16:
start_container(_array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM);
case CS_ARRAY_32:
/* FIXME security guard */
start_container(_array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM);
case CS_MAP_16:
start_container(_map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY);
case CS_MAP_32:
/* FIXME security guard */
start_container(_map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY);
default:
goto _failed;
}
}
_push:
if(top == 0) { goto _finish; }
c = &stack[top-1];
switch(c->ct) {
case CT_ARRAY_ITEM:
if(msgpack_unpack_callback(_array_item)(user, c->curr, &c->obj, obj) < 0) { goto _failed; }
if(++c->curr == c->count) {
msgpack_unpack_callback(_array_end)(user, &c->obj);
obj = c->obj;
--top;
/*printf("stack pop %d\n", top);*/
goto _push;
}
goto _header_again;
case CT_MAP_KEY:
c->map_key = obj;
c->ct = CT_MAP_VALUE;
goto _header_again;
case CT_MAP_VALUE:
if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; }
if(--c->count == 0) {
msgpack_unpack_callback(_map_end)(user, &c->obj);
obj = c->obj;
--top;
/*printf("stack pop %d\n", top);*/
goto _push;
}
c->ct = CT_MAP_KEY;
goto _header_again;
default:
goto _failed;
}
_header_again:
cs = CS_HEADER;
++p;
} while(p != pe);
goto _out;
_finish:
stack[0].obj = obj;
++p;
ret = 1;
/*printf("-- finish --\n"); */
goto _end;
_failed:
/*printf("** FAILED **\n"); */
ret = -1;
goto _end;
_out:
ret = 0;
goto _end;
_end:
ctx->cs = cs;
ctx->trail = trail;
ctx->top = top;
*off = p - (const unsigned char*)data;
return ret;
}
#undef msgpack_unpack_func
#undef msgpack_unpack_callback
#undef msgpack_unpack_struct
#undef msgpack_unpack_object
#undef msgpack_unpack_user
#undef push_simple_value
#undef push_fixed_value
#undef push_variable_value
#undef again_fixed_trail
#undef again_fixed_trail_if_zero
#undef start_container
#undef NEXT_CS
#undef PTR_CAST_8
#undef PTR_CAST_16
#undef PTR_CAST_32
#undef PTR_CAST_64

53
salt/payload.py
View File

@ -5,20 +5,22 @@ encrypted keys to general payload dynamics and packaging, these happen in here
import cPickle as pickle
import salt.msgpack as msgpack
def package(payload, protocol=2):
def package(payload):
'''
This method for now just wraps pickle.dumps, but it is here so that we can
This method for now just wraps msgpack.dumps, but it is here so that we can
make the serialization a custom option in the future with ease.
'''
return pickle.dumps(payload, protocol)
return msgpack.dumps(payload)
def unpackage(package_):
'''
Unpackages a payload
'''
return pickle.loads(package_)
return msgpack.loads(package_)
def format_payload(enc, **kwargs):
@ -32,3 +34,46 @@ def format_payload(enc, **kwargs):
load[key] = kwargs[key]
payload['load'] = load
return package(payload)
class Serial(object):
'''
Create a serialization object, this object manages all message
serialization in Salt
'''
def __init__(self, opts):
self.opts = opts
self.serial = self.opts.get('serial', 'msgpack')
def loads(self, msg):
'''
Run the correct loads serialization format
'''
if self.serial == 'msgpack':
return msgpack.loads(msg)
elif self.serial == 'pickle':
try:
return pickle.loads(msg)
except:
return msgpack.loads(msg)
def load(self, fn_):
'''
Run the correct serialization to load a file
'''
data = fn_.read()
return self.loads(data)
def dumps(self, msg):
'''
Run the correct dums serialization format
'''
if self.serial == 'pickle':
return pickle.dumps(msg)
else:
return msgpack.dumps(msg)
def dump(self, msg, fn_):
'''
Serialize the correct data into the named file object
'''
fn_.write(self.dumps(msg))

42
setup.py
View File

@ -3,16 +3,27 @@
The setup script for salt
'''
import os
import sys
from glob import glob
from distutils.core import setup, Extension
from distutils.command.sdist import sdist
from distutils import log
from distutils.cmd import Command
from distutils.core import setup
from distutils.extension import Extension
from distutils.sysconfig import get_python_lib, PREFIX
import os
import sys
from salt import __version__
try:
from Cython.Distutils import build_ext
import Cython.Compiler.Main as cython_compiler
have_cython = True
except ImportError:
from distutils.command.build_ext import build_ext
have_cython = False
NAME = 'salt'
VER = __version__
DESC = ('Portable, distributed, remote execution and '
@ -27,6 +38,28 @@ if 'SYSCONFDIR' in os.environ:
else:
etc_path = os.path.join(os.path.dirname(PREFIX), 'etc')
# take care of extension modules.
if have_cython:
sources = ['salt/msgpack/_msgpack.pyx']
class Sdist(sdist):
def __init__(self, *args, **kwargs):
for src in glob('salt/msgpack/*.pyx'):
cython_compiler.compile(glob('msgpack/*.pyx'),
cython_compiler.default_options)
sdist.__init__(self, *args, **kwargs)
else:
sources = ['salt/msgpack/_msgpack.c']
Sdist = sdist
libraries = ['ws2_32'] if sys.platform == 'win32' else []
msgpack_mod = Extension('salt.msgpack._msgpack',
sources=sources,
libraries=libraries,
)
setup(
name=NAME,
version=VER,
@ -34,6 +67,8 @@ setup(
author='Thomas S Hatch',
author_email='thatch45@gmail.com',
url='http://saltstack.org',
cmdclass={'build_ext': build_ext, 'sdist': Sdist},
ext_modules=[msgpack_mod],
classifiers=[
'Programming Language :: Python',
'Programming Language :: Cython',
@ -59,6 +94,7 @@ setup(
'salt.runners',
'salt.states',
'salt.utils',
'salt.msgpack',
],
scripts=['scripts/salt-master',
'scripts/salt-minion',