fixed auto cache update, fixed highstate tests - gathering correctly

salt/modules/saltcheck.py

@@ -32,7 +32,7 @@ def update_master_cache():
     Should be done one time before running tests, and if tests are updated
 
     CLI Example:
-        salt '*' salt_check.update_master_cache
+        salt '*' saltcheck.update_master_cache
     '''
     __salt__['cp.cache_master']()
     return True
@@ -40,9 +40,9 @@ def update_master_cache():
 
 def run_test(**kwargs):
     '''
-    Enables running one salt_check test via cli
+    Enables running one saltcheck test via cli
     CLI Example::
-        salt '*' salt_check.run_test
+        salt '*' saltcheck.run_test
           test='{"module_and_function": "test.echo",
             "assertion": "assertEqual",
             "expected-return": "This works!",
@@ -57,32 +57,11 @@ def run_test(**kwargs):
         return "test must be dictionary"
 
 
-def run_state_tests_old(state):
-    '''
-    Returns the output of running all salt check test for a state
-    CLI Example::
-      salt '*' salt_check.run_state_tests postfix_ubuntu_16_04
-    '''
-    scheck = SaltCheck()
-    paths = scheck.get_state_search_path_list()
-    stl = StateTestLoader(search_paths=paths)
-    sls_list = _get_state_sls(state)
-    sls_paths = stl.convert_sls_to_paths(sls_list)
-    for mypath in sls_paths:
-        stl.add_test_files_for_sls(mypath)
-    stl.load_test_suite()
-    results_dict = {}
-    for key, value in stl.test_dict.items():
-        result = scheck.run_test(value)
-        results_dict[key] = result
-    return {state: results_dict}
-
-
 def run_state_tests(state):
     '''
     Returns the output of running all salt check test for a state
     CLI Example::
-      salt '*' salt_check.run_state_tests postfix_ubuntu_16_04
+      salt '*' saltcheck.run_state_tests postfix_ubuntu_16_04
     '''
     scheck = SaltCheck()
     paths = scheck.get_state_search_path_list()
@@ -98,11 +77,13 @@ def run_state_tests(state):
             result = scheck.run_test(value)
             results_dict[key] = result
         results[state_name] = results_dict
+    passed = 0
+    failed = 0
+    missing_tests = 0
     for state in results:
-        passed = 0
-        failed = 0
         if len(results[state].items()) == 0:
-            results[state]['state test results'] = {'pass': passed, 'fail': failed}
+            # results[state]['test results'] = {'pass': passed, 'fail': failed}
+            missing_tests = missing_tests + 1
         else:
             for dummy, val in results[state].items():
                 if val:
@@ -110,29 +91,81 @@ def run_state_tests(state):
                 elif val.upper().startswith('False'):
                     failed = failed + 1
                 else:
-                    failed = 0
+                    # failed = 0
-                    passed = 0
+                    # passed = 0
-                results[state]['state test results'] = {'pass': passed, 'fail': failed}
+                    pass
+    results['state test results'] = {'passed': passed, 'failed': failed, 'missing_tests': missing_tests}
     return results
 
-
 def run_highstate_tests():
     '''
-    Returns the output of running all salt checks of states that would apply for a highstate
+    Returns the output of running all salt check test for a state
     CLI Example::
-        salt '*' salt_check.run_highstate_tests
+      salt '*' saltcheck.run_highstate_tests
     '''
-    states = _get_top_states()
+    scheck = SaltCheck()
-    all_states = {}
+    paths = scheck.get_state_search_path_list()
-    for sta in states:
+    stl = StateTestLoader(search_paths=paths)
-        log.info("State Name = {}".format(sta))
+    results = {}
-        all_states.update(run_state_tests(sta))
+    sls_list = _get_top_states()
+    #sls_list = _get_state_sls(state)
+    for state_name in sls_list:
+        mypath = stl.convert_sls_to_path(state_name)
+        stl.add_test_files_for_sls(mypath)
+        stl.load_test_suite()
+        results_dict = {}
+        for key, value in stl.test_dict.items():
+            result = scheck.run_test(value)
+            results_dict[key] = result
+        results[state_name] = results_dict
     passed = 0
     failed = 0
-    for state in all_states:
+    missing_tests = 0
-        passed = all_states[state]['state test results']['pass'] + passed
+    for state in results:
-        failed = all_states[state]['state test results']['fail'] + failed
+        if len(results[state].items()) == 0:
-    all_states['Total Pass/Fail:'] = {'pass': passed, 'fail': failed}
+            # results[state]['test results'] = {'pass': passed, 'fail': failed}
+            missing_tests = missing_tests + 1
+        else:
+            for dummy, val in results[state].items():
+                if val:
+                    passed = passed + 1
+                elif val.upper().startswith('False'):
+                    failed = failed + 1
+                else:
+                    # failed = 0
+                    # passed = 0
+                    pass
+    results['state test results'] = {'passed': passed, 'failed': failed, 'missing_tests': missing_tests}
+    return results
+
+
+def run_highstate_tests_old():
+    '''
+    Returns the output of running all salt checks of states that would apply for a highstate
+    CLI Example::
+        salt '*' saltcheck.run_highstate_tests
+    '''
+    # there is a bug here....b/c a state can deeply nest it is easy to get false numbers in terms of tests run
+    # with results....need to redo this to load in all tests and pass that dict to SaltCheck - similar to run_state_tests
+    states = _get_top_states()
+    all_states = {}
+    passed = 0
+    failed = 0
+    missing_tests = 0
+    for sta in states:
+        out_dict = None
+        out_dict = run_state_tests(sta)
+        log.info("state-test-result: {}".format(out_dict))
+        passed = out_dict['state test results']['passed'] + passed
+        failed = out_dict['state test results']['failed'] + failed
+        missing_tests = out_dict['state test results']['missing_tests'] + missing_tests
+        r = None
+        r = dict(out_dict)
+        del r['state test results']
+        log.info("out_dict_after_removing_state_test_results: {}".format(out_dict))
+        log.info("r_dict: {}".format(r))
+        all_states.update(r)
+    all_states.update({'test_results': {'passed': passed, 'failed': failed, 'missing_tests': missing_tests} })
     return all_states
 
 
@@ -159,17 +192,20 @@ def _is_valid_function(module_name, function):
 
 def _get_top_states():
     ''' equivalent to a salt cli: salt web state.show_top'''
+    alt_states = []
     try:
         returned = __salt__['state.show_top']()
-        alt_states = []
+        for i in returned['base']:
-        for state in returned['base']:
+            alt_states.append(i)
-            state_bits = state.split(".")
+        #alt_states = []
-            state_name = state_bits[0]
+        #for state in returned['base']:
-            if state_name not in alt_states:
+        #    state_bits = state.split(".")
-                alt_states.append(state_name)
+        #    state_name = state_bits[0]
+        #    if state_name not in alt_states:
+        #        alt_states.append(state_name)
     except Exception:
         raise
-    log.info("top states: {}".format(alt_states))
+    #log.info("top states: {}".format(alt_states))
     return alt_states
 
 
@@ -206,8 +242,8 @@ class SaltCheck(object):
         # log.info("modules are: {}".format(self.modules))
         # self.salt_lc = salt.client.Caller(mopts=__opts__)
         self.salt_lc = salt.client.Caller()
-        # if self.auto_update_master_cache:
+        if self.auto_update_master_cache:
-        #    update_master_cache()
+            update_master_cache()
 
     def __is_valid_test(self, test_dict):
         '''Determine if a test contains:
@@ -256,7 +292,7 @@ class SaltCheck(object):
         return value
 
     def run_test(self, test_dict):
-        '''Run a single salt_check test'''
+        '''Run a single saltcheck test'''
         if self.__is_valid_test(test_dict):
             mod_and_func = test_dict['module_and_function']
             args = test_dict.get('args', None)