Merge pull request #407 from mb0/test-cmd

Test cmd and bug #403 fix

salt/modules/hosts.py

@@ -15,7 +15,7 @@ def list_hosts():
 
         salt '*' hosts.list_hosts
     '''
-    hfn = '/etc/hosts'
+    hfn = list_hosts.hosts_filename
     ret = {}
     if not os.path.isfile(hfn):
         return ret
@@ -26,8 +26,13 @@ def list_hosts():
         if line.startswith('#'):
             continue
         comps = line.split()
-        ret[comps[0]] = comps[1:]
+        if comps[0] in ret:
+            # maybe log a warning ?
+            ret[comps[0]].extend(comps[1:])
+        else:
+            ret[comps[0]] = comps[1:]
     return ret
+list_hosts.hosts_filename = '/etc/hosts'
 
 
 def get_ip(host):
@@ -78,13 +83,13 @@ def has_pair(ip, alias):
 
 def set_host(ip, alias):
     '''
-    Set the host entry in th hosts file for the given ip, this will overwrite
+    Set the host entry in the hosts file for the given ip, this will overwrite
     any previous entry for the given ip
 
     CLI Example::
         salt '*' hosts.set_host <ip> <alias>
     '''
-    hfn = '/etc/hosts'
+    hfn = set_host.hosts_filename
     ovr = False
     if not os.path.isfile(hfn):
         return False
@@ -97,13 +102,20 @@ def set_host(ip, alias):
             continue
         comps = tmpline.split()
         if comps[0] == ip:
-            lines[ind] = ip + '\t\t' + alias + '\n'
-            ovr = True
+            if not ovr:
+                lines[ind] = ip + '\t\t' + alias + '\n'
+                ovr = True
+            else: # remove other entries
+                lines[ind] = ''
     if not ovr:
+        # make sure there is a newline
+        if lines and not lines[-1].endswith(('\n', '\r')):
+            lines[-1] = '%s\n' % lines[-1]
         line = ip + '\t\t' + alias + '\n'
         lines.append(line)
     open(hfn, 'w+').writelines(lines)
     return True
+set_host.hosts_filename = '/etc/hosts'
 
 
 def rm_host(ip, alias):
@@ -115,7 +127,7 @@ def rm_host(ip, alias):
     '''
     if not has_pair(ip, alias):
         return True
-    hfn = '/etc/hosts'
+    hfn = rm_host.hosts_filename
     lines = open(hfn).readlines()
     for ind in range(len(lines)):
         tmpline = lines[ind].strip()
@@ -136,6 +148,7 @@ def rm_host(ip, alias):
                 lines[ind] = newline
     open(hfn, 'w+').writelines(lines)
     return True
+rm_host.hosts_filename = '/etc/hosts'
 
 
 def add_host(ip, alias):
@@ -146,7 +159,7 @@ def add_host(ip, alias):
     CLI Example::
         salt '*' hosts.add_host <ip> <alias>
     '''
-    hfn = '/etc/hosts'
+    hfn = add_host.hosts_filename
     ovr = False
     if not os.path.isfile(hfn):
         return False
@@ -165,8 +178,14 @@ def add_host(ip, alias):
             newline += '\t' + alias
             lines.append(newline)
             ovr = True
+            # leave any other matching entries alone
+            break
     if not ovr:
+        # make sure there is a newline
+        if lines and not lines[-1].endswith(('\n', '\r')):
+            lines[-1] = '%s\n' % lines[-1]
         line = ip + '\t\t' + alias + '\n'
         lines.append(line)
     open(hfn, 'w+').writelines(lines)
     return True
+add_host.hosts_filename = '/etc/hosts'

setup.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/env python
 '''
 The setup script for salt
 '''
@@ -8,13 +8,29 @@ import sys
 from glob import glob
 from distutils.core import setup, Extension
 from distutils.command.sdist import sdist
-from distutils import log
 from distutils.cmd import Command
-from distutils.core import setup
 from distutils.sysconfig import get_python_lib, PREFIX
-
 from salt import __version__
 
+class TestCommand(Command):
+    description = 'Run tests'
+    user_options = []
+    def initialize_options(self): pass
+    def finalize_options(self): pass
+    def run(self):
+        from subprocess import Popen
+        self.run_command('build')
+        build_cmd = self.get_finalized_command('build_ext')
+        runner = os.path.abspath('tests/runtests.py')
+        test_cmd = 'python %s' % runner
+        print("running test")
+        test_process = Popen(
+            test_cmd, shell=True,
+            stdout=sys.stdout, stderr=sys.stderr,
+            cwd=build_cmd.build_lib
+        )
+        test_process.communicate()
+
 try:
     from Cython.Distutils import build_ext
     import Cython.Compiler.Main as cython_compiler
@@ -67,8 +83,8 @@ setup(
       author='Thomas S Hatch',
       author_email='thatch45@gmail.com',
       url='http://saltstack.org',
-      cmdclass={'build_ext': build_ext, 'sdist': Sdist},
       ext_modules=[msgpack_mod],
+      cmdclass={'build_ext': build_ext, 'sdist': Sdist, 'test': TestCommand},
       classifiers=[
           'Programming Language :: Python',
           'Programming Language :: Cython',

tests/modules/__init__.py

@@ -0,0 +1,13 @@
+from salt.cli.caller import Caller
+from salt.config import minion_config
+
+defaults = {
+    'module_dirs': '',
+    'log_level': 'warning',
+}
+config = minion_config('/etc/salt/minion')
+
+def run_module(name, *args):
+    opts = defaults.copy()
+    opts.update(config, fun=name, arg=args)
+    return Caller(opts).call()

tests/modules/files/hosts

@@ -0,0 +1,11 @@
+# a comment
+127.0.0.1      localhost
+# second alias for same ip. 'man hosts' does not allow it
+# but the ubuntu and probably other distributions treat it as
+# 127.0.0.1 localhost myname
+127.0.0.1      myname
+::1     ip6-localhost ip6-loopback
+fe00::0 ip6-localnet
+ff00::0 ip6-mcastprefix
+ff02::1 ip6-allnodes
+ff02::2 ip6-allrouters

tests/modules/hosts.py

@@ -0,0 +1,89 @@
+import unittest
+from salt.modules.hosts import list_hosts, get_ip, get_alias, has_pair, add_host,\
+    set_host, rm_host
+from os import path
+import os
+import shutil
+
+TEMPLATES_DIR = path.dirname(path.abspath(__file__))
+
+monkey_pathed = (list_hosts, set_host, add_host, rm_host)
+class HostsModuleTest(unittest.TestCase):
+    def setUp(self):
+        self._hfn = [f.hosts_filename for f in monkey_pathed]
+        self.files = path.join(TEMPLATES_DIR, 'files')
+        self.hostspath = path.join(self.files, 'hosts')
+        self.not_found = path.join(self.files, 'not_found')
+        self.tmpfiles = []
+
+    def tearDown(self):
+        for i, f in enumerate(monkey_pathed):
+            f.hosts_filename = self._hfn[i]
+        for tmp in self.tmpfiles:
+            os.remove(tmp)
+
+    def tmp_hosts_file(self, src):
+        tmpfile = path.join(self.files, 'tmp')
+        self.tmpfiles.append(tmpfile)
+        shutil.copy(src, tmpfile)
+        return tmpfile
+
+    def test_list_hosts(self):
+        list_hosts.hosts_filename = self.hostspath
+        hosts = list_hosts()
+        self.assertEqual(len(hosts), 6)
+        self.assertEqual(hosts['::1'], ['ip6-localhost', 'ip6-loopback'])
+        self.assertEqual(hosts['127.0.0.1'], ['localhost', 'myname'])
+
+    def test_list_hosts_nofile(self):
+        list_hosts.hosts_filename = self.not_found
+        hosts = list_hosts()
+        self.assertEqual(hosts, {})
+
+    def test_get_ip(self):
+        list_hosts.hosts_filename = self.hostspath
+        self.assertEqual(get_ip('myname'), '127.0.0.1')
+        self.assertEqual(get_ip('othername'), '')
+        list_hosts.hosts_filename = self.not_found
+        self.assertEqual(get_ip('othername'), '')
+
+    def test_get_alias(self):
+        list_hosts.hosts_filename = self.hostspath
+        self.assertEqual(get_alias('127.0.0.1'), ['localhost', 'myname'])
+        self.assertEqual(get_alias('127.0.0.2'), [])
+        list_hosts.hosts_filename = self.not_found
+        self.assertEqual(get_alias('127.0.0.1'), [])
+
+    def test_has_pair(self):
+        list_hosts.hosts_filename = self.hostspath
+        self.assertTrue(has_pair('127.0.0.1', 'myname'))
+        self.assertFalse(has_pair('127.0.0.1', 'othername'))
+
+    def test_set_host(self):
+        tmp = self.tmp_hosts_file(self.hostspath)
+        list_hosts.hosts_filename = tmp
+        set_host.hosts_filename = tmp
+        assert set_host('192.168.1.123', 'newip')
+        self.assertTrue(has_pair('192.168.1.123', 'newip'))
+        self.assertEqual(len(list_hosts()), 7)
+        assert set_host('127.0.0.1', 'localhost')
+        self.assertFalse(has_pair('127.0.0.1', 'myname'), 'should remove second entry')
+
+    def test_add_host(self):
+        tmp = self.tmp_hosts_file(self.hostspath)
+        list_hosts.hosts_filename = tmp
+        add_host.hosts_filename = tmp
+        assert add_host('192.168.1.123', 'newip')
+        self.assertTrue(has_pair('192.168.1.123', 'newip'))
+        self.assertEqual(len(list_hosts()), 7)
+        assert add_host('127.0.0.1', 'othernameip')
+        self.assertEqual(len(list_hosts()), 7)
+
+    def test_rm_host(self):
+        tmp = self.tmp_hosts_file(self.hostspath)
+        list_hosts.hosts_filename = tmp
+        rm_host.hosts_filename = tmp
+        assert has_pair('127.0.0.1', 'myname')
+        assert rm_host('127.0.0.1', 'myname')
+        assert not has_pair('127.0.0.1', 'myname')
+        assert rm_host('127.0.0.1', 'unknown')

tests/modules/test.py

@@ -0,0 +1,7 @@
+import unittest
+from modules import run_module
+
+class TestModuleTest(unittest.TestCase):
+    def test_ping(self):
+        ret = run_module('test.ping')
+        assert ret == {'return': True}

tests/runtests.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+'''
+Discover all instances of unittest.TestCase in this directory.
+
+The current working directory must be set to the build of the salt you want to test.
+'''
+from unittest import TestLoader, TextTestRunner
+from os.path import dirname, abspath, relpath, splitext, normpath
+import sys, os, fnmatch
+
+TEST_DIR = dirname(normpath(abspath(__file__)))
+SALT_BUILD = os.getcwd()
+TEST_FILES = '*.py'
+
+sys.path.insert(0, TEST_DIR)
+sys.path.insert(0, SALT_BUILD)
+
+def main():
+    names = find_tests()
+    tests = TestLoader().loadTestsFromNames(names)
+    TextTestRunner(verbosity=1).run(tests)
+
+def find_tests():
+    names = []
+    for root, _, files in os.walk(TEST_DIR):
+        for name in files:
+            if fnmatch.fnmatch(name, TEST_FILES) \
+                    and not name == 'runtests.py':
+                module = get_test_name(root, name)
+                if module: names.append(module)
+    return names
+def get_test_name(root, name):
+    if name.startswith("_"): return None
+    rel = relpath(root, TEST_DIR).lstrip(".")
+    prefix = "%s." % rel.replace('/','.') if rel else ""
+    return "".join((prefix, splitext(name)[0]))
+
+if __name__ == "__main__":
+    main()

tests/simple.py

@@ -0,0 +1,8 @@
+import unittest
+
+class SimpleTest(unittest.TestCase):
+    def test_success(self):
+        assert True
+    @unittest.expectedFailure
+    def test_fail(self):
+        assert False

tests/templates/files/test/hello_import

@@ -0,0 +1,3 @@
+{% from 'macro' import mymacro -%}
+{% from 'macro' import mymacro -%}
+{{ mymacro('Hey') ~ mymacro(a|default('a'), b|default('b')) }}

tests/templates/files/test/hello_include

@@ -0,0 +1 @@
+{% include 'hello_import' -%}

tests/templates/files/test/hello_simple

@@ -0,0 +1 @@
+world

tests/templates/files/test/macro

@@ -0,0 +1,4 @@
+# macro
+{% macro mymacro(greeting, greetee='world') -%}
+{{ greeting ~ ' ' ~ greetee }} !
+{%- endmacro %}

tests/templates/jinja.py

@@ -0,0 +1,112 @@
+import unittest
+from os import path
+from salt.utils.jinja import SaltCacheLoader, get_template
+from jinja2 import Environment
+
+TEMPLATES_DIR = path.dirname(path.abspath(__file__))
+
+class MockFileClient(object):
+    '''
+    Does not download files but records any file request for testing
+    '''
+    def __init__(self, loader=None):
+        if loader: loader._file_client = self
+        self.requests = []
+    def get_file(self, template, dest='', makedirs=False, env='base'):
+        self.requests.append({
+            'path': template, 
+            'dest': dest,
+            'makedirs': makedirs,
+            'env': env
+        })
+
+class TestSaltCacheLoader(unittest.TestCase):
+    def test_searchpath(self):
+        '''
+        The searchpath is based on the cachedir option and the env parameter
+        '''
+        loader = SaltCacheLoader({'cachedir': '/tmp'}, env='test')
+        assert loader.searchpath == '/tmp/files/test'
+    def test_mockclient(self):
+        '''
+        A MockFileClient is used that records all file request normally send to the master.
+        '''
+        loader = SaltCacheLoader({'cachedir': TEMPLATES_DIR}, 'test')
+        fc = MockFileClient(loader)
+        res = loader.get_source(None, 'hello_simple')
+        assert len(res) == 3
+        self.assertEqual(res[0], 'world\n')
+        self.assertEqual(res[1], '%s/files/test/hello_simple' % TEMPLATES_DIR)
+        assert res[2](), "Template up to date?"
+        assert len(fc.requests)
+        self.assertEqual(fc.requests[0]['path'], 'salt://hello_simple')
+    def get_test_env(self):
+        '''
+        Setup a simple jinja test environment
+        '''
+        loader = SaltCacheLoader({'cachedir': TEMPLATES_DIR}, 'test')
+        fc = MockFileClient(loader)
+        jinja = Environment(loader=loader)
+        return fc, jinja
+    def test_import(self):
+        '''
+        You can import and use macros from other files
+        '''
+        fc, jinja = self.get_test_env()
+        result = jinja.get_template('hello_import').render()
+        self.assertEqual(result, 'Hey world !a b !')
+        assert len(fc.requests) == 2
+        self.assertEqual(fc.requests[0]['path'], 'salt://hello_import')
+        self.assertEqual(fc.requests[1]['path'], 'salt://macro')
+    def test_include(self):
+        '''
+        You can also include a template that imports and uses macros
+        '''
+        fc, jinja = self.get_test_env()
+        result = jinja.get_template('hello_include').render()
+        self.assertEqual(result, 'Hey world !a b !')
+        assert len(fc.requests) == 3
+        self.assertEqual(fc.requests[0]['path'], 'salt://hello_include')
+        self.assertEqual(fc.requests[1]['path'], 'salt://hello_import')
+        self.assertEqual(fc.requests[2]['path'], 'salt://macro')
+    def test_include_context(self):
+        '''
+        Context variables are passes to the included template by default.
+        '''
+        _, jinja = self.get_test_env()
+        result = jinja.get_template('hello_include').render(a='Hi', b='Salt')
+        self.assertEqual(result, 'Hey world !Hi Salt !')
+
+class TestGetTemplate(unittest.TestCase):
+    def test_fallback(self):
+        '''
+        A Template without loader is returned as fallback 
+        if the file is not contained in the searchpath
+        '''
+        filename = '%s/files/test/hello_simple' % TEMPLATES_DIR
+        tmpl = get_template(filename, {'cachedir': TEMPLATES_DIR}, env='other')
+        self.assertEqual(tmpl.render(), 'world')
+    def test_fallback_noloader(self):
+        '''
+        If the fallback is used any attempt to load other templates
+        will raise a TypeError.
+        '''
+        filename = '%s/files/test/hello_import' % TEMPLATES_DIR
+        tmpl = get_template(filename, {'cachedir': TEMPLATES_DIR}, env='other')
+        self.assertRaises(TypeError, tmpl.render)
+    def test_env(self):
+        '''
+        If the template is within the searchpath it can
+        import, include and extend other templates.
+        The initial template is expected to be already cached
+        get_template does not request it from the master again.
+        '''
+        fc = MockFileClient()
+        # monkey patch file client
+        _fc = SaltCacheLoader.file_client
+        SaltCacheLoader.file_client = lambda loader: fc 
+        filename = '%s/files/test/hello_import' % TEMPLATES_DIR
+        tmpl = get_template(filename, {'cachedir': TEMPLATES_DIR}, env='test')
+        self.assertEqual(tmpl.render(a='Hi', b='Salt'), 'Hey world !Hi Salt !')
+        self.assertEqual(fc.requests[0]['path'], 'salt://macro')
+        SaltCacheLoader.file_client = _fc