From 83b8fdac0ff52f1f1d9f96772b956c807f668593 Mon Sep 17 00:00:00 2001
From: David Reiss <dreiss@apache.org>
Date: Tue, 9 Mar 2010 05:19:34 +0000
Subject: [PATCH] cpp: Fix for task completion notification deadlock under
 heavy server load

Along with insuring that the event loop is able to see task completion
notifications, we need to process pending notifications as soon as they
are available to avoid filling the notification pipe (the pipe can hold
512 notifications).

git-svn-id: https://svn.apache.org/repos/asf/incubator/thrift/trunk@920668 13f79535-47bb-0310-9956-ffa450edef68
---
 lib/cpp/src/server/TNonblockingServer.cpp | 11 +++++++++--
 lib/cpp/src/server/TNonblockingServer.h   | 16 ++++++++++------
 2 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/lib/cpp/src/server/TNonblockingServer.cpp b/lib/cpp/src/server/TNonblockingServer.cpp
index 26e97c967..eb071a903 100644
--- a/lib/cpp/src/server/TNonblockingServer.cpp
+++ b/lib/cpp/src/server/TNonblockingServer.cpp
@@ -570,12 +570,12 @@ void TNonblockingServer::handleEvent(int fd, short which) {
       nTotalConnectionsDropped_++;
       if (overloadAction_ == T_OVERLOAD_CLOSE_ON_ACCEPT) {
         close(clientSocket);
-        continue;
+        return;
       } else if (overloadAction_ == T_OVERLOAD_DRAIN_TASK_QUEUE) {
         if (!drainPendingTask()) {
           // Nothing left to discard, so we drop connection instead.
           close(clientSocket);
-          continue;
+          return;
         }
       }
     }
@@ -718,6 +718,13 @@ void TNonblockingServer::createNotificationPipe() {
     GlobalOutput.perror("TNonblockingServer::createNotificationPipe ", errno);
       throw TException("can't create notification pipe");
   }
+  int flags;
+  if ((flags = fcntl(notificationPipeFDs_[0], F_GETFL, 0)) < 0 ||
+      fcntl(notificationPipeFDs_[0], F_SETFL, flags | O_NONBLOCK) < 0) {
+    close(notificationPipeFDs_[0]);
+    close(notificationPipeFDs_[1]);
+    throw TException("TNonblockingServer::createNotificationPipe() O_NONBLOCK");
+  }
 }
 
 /**
diff --git a/lib/cpp/src/server/TNonblockingServer.h b/lib/cpp/src/server/TNonblockingServer.h
index 2650dd119..b54764966 100644
--- a/lib/cpp/src/server/TNonblockingServer.h
+++ b/lib/cpp/src/server/TNonblockingServer.h
@@ -722,13 +722,17 @@ class TConnection {
    */
   static void taskHandler(int fd, short /* which */, void* /* v */) {
     TConnection* connection;
-    if (read(fd, (void*)&connection, sizeof(TConnection*))
-        != sizeof(TConnection*)) {
-      GlobalOutput.perror("TConnection::taskHandler read failed, resource leak", errno);
-      return;
+    ssize_t nBytes;
+    while ((nBytes = read(fd, (void*)&connection, sizeof(TConnection*)))
+        == sizeof(TConnection*)) {
+      connection->transition();
+    }
+    if (nBytes > 0) {
+      throw TException("TConnection::taskHandler unexpected partial read");
+    }
+    if (errno != EWOULDBLOCK && errno != EAGAIN) {
+      GlobalOutput.perror("TConnection::taskHandler read failed, resource leak", errno);
     }
-
-    connection->transition();
   }
 
   /**