THRIFT-3571 Make feature test result browsable

Client: Test
Patch: Nobuaki Sukegawa

This closes #809

.gitignore

@@ -240,6 +240,7 @@ erl_crash.dump
 /missing
 /node_modules/
 /stamp-h1
+/test/features/results.json
 /test/results.json
 /test/c_glib/test_client
 /test/c_glib/test_server

Makefile.am

@@ -55,9 +55,15 @@ else
 CROSS_PY=$(PYTHON)
 endif
 
+if WITH_PYTHON
 cross: precross
 	$(CROSS_PY) test/test.py -F.* -s --server $(CROSS_LANGS_COMMA_SEPARATED)
 	$(CROSS_PY) test/test.py -s --server $(CROSS_LANGS_COMMA_SEPARATED) --client $(CROSS_LANGS_COMMA_SEPARATED)
+else
+# feature test needs python build
+cross: precross
+	$(CROSS_PY) test/test.py -s --server $(CROSS_LANGS_COMMA_SEPARATED) --client $(CROSS_LANGS_COMMA_SEPARATED)
+endif
 
 TIMES = 1 2 3
 fail: precross

configure.ac

@@ -740,6 +740,7 @@ AC_CONFIG_FILES([
   lib/xml/Makefile
   lib/xml/test/Makefile
   test/Makefile
+  test/features/Makefile
   test/c_glib/Makefile
   test/cpp/Makefile
   test/erl/Makefile

test/Makefile.am

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-SUBDIRS =
+SUBDIRS = features
 PRECROSS_TARGET =
 
 if WITH_C_GLIB
@@ -124,7 +124,8 @@ EXTRA_DIST = \
 	tests.json \
 	ThriftTest.thrift \
 	TypedefTest.thrift \
-	result.html \
+	result.js \
+	index.html \
 	README.md \
 	valgrind.suppress
 

test/crossrunner/report.py

@@ -33,7 +33,7 @@ from .compat import logfile_open, path_join, str_join
 from .test import TestEntry
 
 LOG_DIR = 'log'
-RESULT_HTML = 'result.html'
+RESULT_HTML = 'index.html'
 RESULT_JSON = 'results.json'
 FAIL_JSON = 'known_failures_%s.json'
 
@@ -209,11 +209,12 @@ class ExecReporter(TestReporter):
 
 
 class SummaryReporter(TestReporter):
-  def __init__(self, testdir, concurrent=True):
+  def __init__(self, basedir, testdir_relative, concurrent=True):
     super(SummaryReporter, self).__init__()
-    self.testdir = testdir
-    self.logdir = path_join(testdir, LOG_DIR)
-    self.out_path = path_join(testdir, RESULT_JSON)
+    self._basedir = basedir
+    self._testdir_rel = testdir_relative
+    self.logdir = path_join(self.testdir, LOG_DIR)
+    self.out_path = path_join(self.testdir, RESULT_JSON)
     self.concurrent = concurrent
     self.out = sys.stdout
     self._platform = platform.system()
@@ -221,12 +222,16 @@ class SummaryReporter(TestReporter):
     self._tests = []
     if not os.path.exists(self.logdir):
       os.mkdir(self.logdir)
-    self._known_failures = load_known_failures(testdir)
+    self._known_failures = load_known_failures(self.testdir)
     self._unexpected_success = []
     self._unexpected_failure = []
     self._expected_failure = []
     self._print_header()
 
+  @property
+  def testdir(self):
+    return path_join(self._basedir, self._testdir_rel)
+
   def _get_revision(self):
     p = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'],
                          cwd=self.testdir, stdout=subprocess.PIPE)
@@ -296,11 +301,11 @@ class SummaryReporter(TestReporter):
     self._assemble_log('known failures', self._expected_failure)
     self.out.writelines([
       'You can browse results at:\n',
-      '\tfile://%s/%s\n' % (self.testdir, RESULT_HTML),
+      '\tfile://%s/%s\n' % (self._basedir, RESULT_HTML),
       '# If you use Chrome, run:\n',
-      '# \tcd %s\n#\t%s\n' % (self.testdir, self._http_server_command(8001)),
+      '# \tcd %s\n#\t%s\n' % (self._basedir, self._http_server_command(8001)),
       '# then browse:\n',
-      '# \thttp://localhost:%d/test/%s\n' % (8001, RESULT_HTML),
+      '# \thttp://localhost:%d/%s/\n' % (8001, self._testdir_rel),
       'Full log for each test is here:\n',
       '\ttest/log/client_server_protocol_transport_client.log\n',
       '\ttest/log/client_server_protocol_transport_server.log\n',

test/crossrunner/run.py

@@ -269,10 +269,11 @@ class NonAsyncResult(object):
 
 
 class TestDispatcher(object):
-  def __init__(self, testdir, logdir, concurrency):
+  def __init__(self, testdir, basedir, logdir_rel, concurrency):
     self._log = multiprocessing.get_logger()
     self.testdir = testdir
-    self.logdir = logdir
+    self._report = SummaryReporter(basedir, logdir_rel, concurrency > 1)
+    self.logdir = self._report.testdir
     # seems needed for python 2.x to handle keyboard interrupt
     self._stop = multiprocessing.Event()
     self._async = concurrency > 1
@@ -287,7 +288,6 @@ class TestDispatcher(object):
       self._m.register('ports', PortAllocator)
       self._m.start()
       self._pool = multiprocessing.Pool(concurrency, self._pool_init, (self._m.address,))
-    self._report = SummaryReporter(logdir, concurrency > 1)
     self._log.debug(
         'TestDispatcher started with %d concurrent jobs' % concurrency)
 

test/features/Makefile.am

@@ -0,0 +1,27 @@
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+EXTRA_DIST = \
+	local_thrift \
+	index.html \
+	container_limit.py \
+	index.html \
+	known_failures_Linux.json \
+	Makefile.am \
+	string_limit.py \
+	tests.json \
+	theader_binary.py \
+	util.py

test/features/index.html

@@ -0,0 +1,51 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<!DOCTYPE HTML>
+<html>
+<head>
+<meta charset="utf-8">
+<title>Apache Thrift - integration test suite</title>
+<link rel="stylesheet" type="text/css" href="http://cdn.datatables.net/1.10.4/css/jquery.dataTables.css">
+<script type="text/javascript" charset="utf-8" src="http://code.jquery.com/jquery-2.1.3.min.js"></script>
+<script type="text/javascript" charset="utf-8" src="http://cdn.datatables.net/1.10.4/js/jquery.dataTables.js"></script>
+<script src="../result.js">
+</script>
+</head>
+<body>
+<h2>Apache Thrift - integration test suite: Results</h2>
+<table id="test_results" class="display">
+    <thead>
+        <tr>
+            <th>Server</th>
+            <th>Client</th>
+            <th>Protocol</th>
+            <th>Transport</th>
+            <th>Result (log)</th>
+            <th>Expected</th>
+        </tr>
+    </thead>
+</table>
+<h2>Test Information</h2>
+<pre id="test_info"></pre>
+
+<a href="log">browse raw log files</a>
+
+</body>
+</html>

test/index.html

@@ -0,0 +1,51 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<!DOCTYPE HTML>
+<html>
+<head>
+<meta charset="utf-8">
+<title>Apache Thrift - integration test suite</title>
+<link rel="stylesheet" type="text/css" href="http://cdn.datatables.net/1.10.4/css/jquery.dataTables.css">
+<script type="text/javascript" charset="utf-8" src="http://code.jquery.com/jquery-2.1.3.min.js"></script>
+<script type="text/javascript" charset="utf-8" src="http://cdn.datatables.net/1.10.4/js/jquery.dataTables.js"></script>
+<script src="result.js">
+</script>
+</head>
+<body>
+<h2>Apache Thrift - integration test suite: Results</h2>
+<table id="test_results" class="display">
+    <thead>
+        <tr>
+            <th>Server</th>
+            <th>Client</th>
+            <th>Protocol</th>
+            <th>Transport</th>
+            <th>Result (log)</th>
+            <th>Expected</th>
+        </tr>
+    </thead>
+</table>
+<h2>Test Information</h2>
+<pre id="test_info"></pre>
+
+<a href="log">browse raw log</a>
+
+</body>
+</html>

test/result.js

@@ -1,4 +1,4 @@
-<!--
+/*
  Licensed to the Apache Software Foundation (ASF) under one
  or more contributor license agreements. See the NOTICE file
  distributed with this work for additional information
@@ -16,16 +16,8 @@
  specific language governing permissions and limitations
  under the License.
 
--->
-<!DOCTYPE HTML>
-<html>
-<head>
-<meta charset="utf-8">
-<title>Apache Thrift - integration test suite</title>
-<link rel="stylesheet" type="text/css" href="http://cdn.datatables.net/1.10.4/css/jquery.dataTables.css">
-<script type="text/javascript" charset="utf-8" src="http://code.jquery.com/jquery-2.1.3.min.js"></script>
-<script type="text/javascript" charset="utf-8" src="http://cdn.datatables.net/1.10.4/js/jquery.dataTables.js"></script>
-<script>
+*/
+
 $.getJSON('results.json', function(results) {
     $(document).ready(function() {
         var transport = 3;
@@ -69,23 +61,4 @@ $.getJSON('results.json', function(results) {
             "Test duration: " + results['duration']) + " seconds";
     });
 });
-</script>
-</head>
-<body>
-<h2>Apache Thrift - integration test suite: Results</h2>
-<table id="test_results" class="display">
-    <thead>
-        <tr>
-            <th>Server</th>
-            <th>Client</th>
-            <th>Protocol</th>
-            <th>Transport</th>
-            <th>Result (log)</th>
-            <th>Expected</th>
-        </tr>
-    </thead>
-</table>
-<h2>Test Information</h2>
-<pre id="test_info"></pre>
-</body>
-</html>
+

test/test.py

@@ -38,40 +38,13 @@ import sys
 import crossrunner
 from crossrunner.compat import path_join
 
-TEST_DIR = os.path.realpath(os.path.dirname(__file__))
-FEATURE_DIR = path_join(TEST_DIR, 'features')
+ROOT_DIR = os.path.dirname(os.path.realpath(os.path.dirname(__file__)))
+TEST_DIR_RELATIVE = 'test'
+TEST_DIR = path_join(ROOT_DIR, TEST_DIR_RELATIVE)
+FEATURE_DIR_RELATIVE = path_join(TEST_DIR_RELATIVE, 'features')
 CONFIG_FILE = 'tests.json'
 
 
-def run_tests(collect_func, basedir, server_match, client_match, jobs, skip):
-  logger = multiprocessing.get_logger()
-  logger.debug('Collecting tests')
-  with open(path_join(basedir, CONFIG_FILE), 'r') as fp:
-    j = json.load(fp)
-  tests = collect_func(j, server_match, client_match)
-  if not tests:
-    print('No test found that matches the criteria', file=sys.stderr)
-    # print('  servers: %s' % server_match, file=sys.stderr)
-    # print('  clients: %s' % client_match, file=sys.stderr)
-    return False
-  if skip:
-    logger.debug('Skipping known failures')
-    known = crossrunner.load_known_failures(basedir)
-    tests = list(filter(lambda t: crossrunner.test_name(**t) not in known, tests))
-
-  dispatcher = crossrunner.TestDispatcher(TEST_DIR, basedir, jobs)
-  logger.debug('Executing %d tests' % len(tests))
-  try:
-    for r in [dispatcher.dispatch(test) for test in tests]:
-      r.wait()
-    logger.debug('Waiting for completion')
-    return dispatcher.wait()
-  except (KeyboardInterrupt, SystemExit):
-    logger.debug('Interrupted, shutting down')
-    dispatcher.terminate()
-    return False
-
-
 def run_cross_tests(server_match, client_match, jobs, skip_known_failures):
   logger = multiprocessing.get_logger()
   logger.debug('Collecting tests')
@@ -88,7 +61,7 @@ def run_cross_tests(server_match, client_match, jobs, skip_known_failures):
     known = crossrunner.load_known_failures(TEST_DIR)
     tests = list(filter(lambda t: crossrunner.test_name(**t) not in known, tests))
 
-  dispatcher = crossrunner.TestDispatcher(TEST_DIR, TEST_DIR, jobs)
+  dispatcher = crossrunner.TestDispatcher(TEST_DIR, ROOT_DIR, TEST_DIR_RELATIVE, jobs)
   logger.debug('Executing %d tests' % len(tests))
   try:
     for r in [dispatcher.dispatch(test) for test in tests]:
@@ -102,7 +75,7 @@ def run_cross_tests(server_match, client_match, jobs, skip_known_failures):
 
 
 def run_feature_tests(server_match, feature_match, jobs, skip_known_failures):
-  basedir = FEATURE_DIR
+  basedir = path_join(ROOT_DIR, FEATURE_DIR_RELATIVE)
   logger = multiprocessing.get_logger()
   logger.debug('Collecting tests')
   with open(path_join(TEST_DIR, CONFIG_FILE), 'r') as fp:
@@ -120,7 +93,7 @@ def run_feature_tests(server_match, feature_match, jobs, skip_known_failures):
     known = crossrunner.load_known_failures(basedir)
     tests = list(filter(lambda t: crossrunner.test_name(**t) not in known, tests))
 
-  dispatcher = crossrunner.TestDispatcher(TEST_DIR, basedir, jobs)
+  dispatcher = crossrunner.TestDispatcher(TEST_DIR, ROOT_DIR, FEATURE_DIR_RELATIVE, jobs)
   logger.debug('Executing %d tests' % len(tests))
   try:
     for r in [dispatcher.dispatch(test) for test in tests]:
@@ -179,7 +152,7 @@ def main(argv):
   client_match = list(chain(*[x.split(',') for x in options.client]))
 
   if options.update_failures or options.print_failures:
-    dire = FEATURE_DIR if options.features is not None else TEST_DIR
+    dire = path_join(ROOT_DIR, FEATURE_DIR_RELATIVE) if options.features is not None else TEST_DIR
     res = crossrunner.generate_known_failures(
         dire, options.update_failures == 'overwrite',
         options.update_failures, options.print_failures)