[2212] make sure catching internal error exceptions.

with fixing trivial conflicts:
	src/bin/auth/tests/datasrc_clients_builder_unittest.cc

src/bin/auth/auth_messages.mes

@@ -349,3 +349,12 @@ This is a debug message output during the processing of a NOTIFY
 request. The zone manager component has been informed of the request,
 but has returned an error response (which is included in the message). The
 NOTIFY request will not be honored.
+
+% AUTH_DATASRC_CLIENTS_BUILDER_COMMAND_ERROR command execution failure: %1
+The separate thread for maintaining data source clients failed to complete
+a comment given by the main thread.  In most cases this is some kind of
+configuration or temporary errors such as an attempt of non existent zone
+or temporary DB connection failure.  So the event is just logged and the
+thread keeps running.  In some rare cases, however, this may indicate
+an internal bug and it may be better to restart the entire program.
+So the log message should be carefully examined.

src/bin/auth/datasrc_clients_mgr.h

@@ -419,7 +419,13 @@ DataSrcClientsBuilderBase<MutexType, CondVarType>::run() {
             } // the lock is released here.
 
             while (keep_running && !current_commands.empty()) {
-                keep_running = handleCommand(current_commands.front());
+                try {
+                    keep_running = handleCommand(current_commands.front());;
+                } catch (const InternalCommandError& e) {
+                    LOG_ERROR(auth_logger,
+                              AUTH_DATASRC_CLIENTS_BUILDER_COMMAND_ERROR).
+                        arg(e.what());
+                }
                 current_commands.pop_front();
             }
         }

src/bin/auth/tests/datasrc_clients_builder_unittest.cc

@@ -104,6 +104,11 @@ TEST_F(DataSrcClientsBuilderTest, exception) {
     command_queue.push_back(noop_cmd);
     queue_mutex.throw_from_noop = TestMutex::INTEGER;
     EXPECT_DEATH_IF_SUPPORTED({builder.run();}, "");
+
+    command_queue.push_back(noop_cmd);
+    command_queue.push_back(shutdown_cmd); // we need to stop the loop
+    queue_mutex.throw_from_noop = TestMutex::INTERNAL;
+    builder.run();
 }
 
 TEST_F(DataSrcClientsBuilderTest, condWait) {

src/bin/auth/tests/test_datasrc_clients_mgr.cc

@@ -50,6 +50,8 @@ TestDataSrcClientsBuilder::doNoop() {
         isc_throw(Exception, "test exception");
     case TestMutex::INTEGER:
         throw 42;
+    case TestMutex::INTERNAL:
+        isc_throw(InternalCommandError, "internal error, should be ignored");
     }
 }
 } // namespace datasrc_clientmgr_internal

src/bin/auth/tests/test_datasrc_clients_mgr.h

@@ -43,7 +43,8 @@ public:
     // None: no throw from specialized doNoop()
     // EXCLASS: throw some exception class object
     // INTEGER: throw an integer
-    enum ExceptionFromNoop { NONE, EXCLASS, INTEGER };
+    // INTERNAL: internal error (shouldn't terminate the thread)
+    enum ExceptionFromNoop { NONE, EXCLASS, INTEGER, INTERNAL };
 
     TestMutex() : lock_count(0), unlock_count(0), noop_count(0),
                   throw_from_noop(NONE)