Limit and reduce the ammount of log records sent over the wire

tests/integration/__init__.py

@@ -964,6 +964,7 @@ class TestDaemon(object):
                 conf['log_handlers_dirs'] = []
             conf['log_handlers_dirs'].insert(0, LOG_HANDLERS_DIR)
             conf['runtests_log_port'] = SALT_LOG_PORT
+            conf['runtests_log_level'] = os.environ.get('TESTS_MIN_LOG_LEVEL_NAME') or 'debug'
 
         # ----- Transcribe Configuration ---------------------------------------------------------------------------->
         for entry in os.listdir(RUNTIME_VARS.CONF_DIR):

tests/integration/files/log_handlers/runtests_log_handler.py

@@ -56,9 +56,15 @@ def setup_handlers():
             pass
         sock.close()
 
-    queue = Queue()
+    # One million log messages is more than enough to queue.
+    # Above that value, if `process_queue` can't process fast enough,
+    # start dropping. This will contain a memory leak in case `process_queue`
+    # can't process fast enough of in case it can't deliver the log records at all.
+    queue_size = 10000000
+    queue = Queue(queue_size)
     handler = salt.log.setup.QueueHandler(queue)
-    handler.setLevel(1)
+    level = salt.log.setup.LOG_LEVELS[(__opts__.get('runtests_log_level') or 'error').lower()]
+    handler.setLevel(level)
     process_queue_thread = threading.Thread(target=process_queue, args=(port, queue))
     process_queue_thread.daemon = True
     process_queue_thread.start()

tests/support/parser/__init__.py

@@ -610,6 +610,9 @@ class SaltTestingParser(optparse.OptionParser):
         # Default logging level: ERROR
         logging.root.setLevel(logging.NOTSET)
 
+        log_levels_to_evaluate = [
+            logging.ERROR,  # Default log level
+        ]
         if self.options.tests_logfile:
             filehandler = logging.FileHandler(
                 mode='w',           # Not preserved between re-runs
@@ -620,6 +623,7 @@ class SaltTestingParser(optparse.OptionParser):
             filehandler.setLevel(logging.DEBUG)
             filehandler.setFormatter(formatter)
             logging.root.addHandler(filehandler)
+            log_levels_to_evaluate.append(logging.DEBUG)
 
             print(' * Logging tests on {0}'.format(self.options.tests_logfile))
 
@@ -633,17 +637,18 @@ class SaltTestingParser(optparse.OptionParser):
                 logging_level = logging.TRACE
             elif self.options.verbosity == 4:   # -vvv
                 logging_level = logging.DEBUG
-                print('DEBUG')
             elif self.options.verbosity == 3:   # -vv
-                print('INFO')
                 logging_level = logging.INFO
             else:
                 logging_level = logging.ERROR
+            log_levels_to_evaluate.append(logging_level)
             os.environ['TESTS_LOG_LEVEL'] = str(self.options.verbosity)  # future lint: disable=blacklisted-function
             consolehandler.setLevel(logging_level)
             logging.root.addHandler(consolehandler)
             log.info('Runtests logging has been setup')
 
+        os.environ['TESTS_MIN_LOG_LEVEL_NAME'] = logging.getLevelName(min(log_levels_to_evaluate))
+
     def pre_execution_cleanup(self):
         '''
         Run any initial clean up operations. If sub-classed, don't forget to