27 #include <boost/pointer_cast.hpp> 28 #include <boost/make_shared.hpp> 29 #include <boost/weak_ptr.hpp> 41 namespace ph = std::placeholders;
48 CommandUnsupportedError(
const char* file,
size_t line,
const char* what) :
57 const int HAService::HA_HEARTBEAT_COMPLETE_EVT;
58 const int HAService::HA_LEASE_UPDATES_COMPLETE_EVT;
59 const int HAService::HA_SYNCING_FAILED_EVT;
60 const int HAService::HA_SYNCING_SUCCEEDED_EVT;
61 const int HAService::HA_MAINTENANCE_NOTIFY_EVT;
62 const int HAService::HA_MAINTENANCE_START_EVT;
63 const int HAService::HA_MAINTENANCE_CANCEL_EVT;
64 const int HAService::HA_CONTROL_RESULT_MAINTENANCE_NOT_ALLOWED;
65 const int HAService::HA_SYNCED_PARTNER_UNAVAILABLE_EVT;
69 : io_service_(io_service), network_state_(network_state), config_(config),
70 server_type_(server_type), client_(), listener_(), communication_state_(),
71 query_filter_(config), mutex_(), pending_requests_(),
72 lease_update_backlog_(config->getDelayedUpdatesLimit()),
73 sync_complete_notified_(false) {
87 if (!
config_->getEnableMultiThreading()) {
93 config_->getHttpClientThreads(),
true));
96 if (
config_->getHttpDedicatedListener()) {
98 auto my_url =
config_->getThisServerConfig()->getUrl();
99 IOAddress server_address(IOAddress::IPV4_ZERO_ADDRESS());
103 server_address =
IOAddress(my_url.getStrippedHostname());
104 }
catch (
const std::exception& ex) {
106 <<
" is not a valid IP address");
110 uint32_t listener_threads =
config_->getHttpListenerThreads();
113 auto tls_context =
config_->getThisServerConfig()->getTlsContext();
117 listener_threads, tls_context));
119 if (
config_->getRestrictCommands()) {
121 CmdResponseCreator::command_accept_list_ =
124 CmdResponseCreator::command_accept_list_ =
145 StateModel::defineEvents();
159 StateModel::verifyEvents();
173 StateModel::defineStates();
412 }
else if (
config_->amAllowingCommRecovery()) {
471 if (maintenance ||
config_->getThisServerConfig()->isAutoFailover()) {
742 unsigned int dhcp_disable_timeout =
743 static_cast<unsigned int>(
config_->getSyncTimeout() / 1000);
744 if (dhcp_disable_timeout == 0) {
745 ++dhcp_disable_timeout;
749 std::string status_message;
751 config_->getFailoverPeerConfig()->getName(),
752 dhcp_disable_timeout);
907 boost::to_upper(current_state_name);
908 boost::to_upper(new_state_name);
914 std::string partner_state_name =
getStateLabel(partner_state);
915 boost::to_upper(partner_state_name);
919 .arg(current_state_name)
921 .arg(partner_state_name);
926 .arg(current_state_name)
927 .arg(new_state_name);
948 .arg(new_state_name);
950 }
else if (!
config_->amSendingLeaseUpdates()) {
953 .arg(new_state_name);
960 .arg(new_state_name);
971 switch (
config_->getHAMode()) {
996 boost::to_upper(state_name);
1009 return (inScopeInternal(query4));
1014 return (inScopeInternal(query6));
1017 template<
typename QueryPtrType>
1019 HAService::inScopeInternal(QueryPtrType& query) {
1021 std::string scope_class;
1040 boost::to_upper(current_state_name);
1053 boost::to_upper(current_state_name);
1055 .arg(
config_->getThisServerName())
1056 .arg(current_state_name);
1057 network_state_->disableService(NetworkState::Origin::HA_COMMAND);
1059 }
else if (should_enable && !
network_state_->isServiceEnabled()) {
1061 boost::to_upper(current_state_name);
1063 .arg(
config_->getThisServerName())
1064 .arg(current_state_name);
1102 if (!should_terminate) {
1106 return (should_terminate);
1153 size_t sent_num = 0;
1156 for (
auto p = peers_configs.begin(); p != peers_configs.end(); ++p) {
1164 for (
auto l = deleted_leases->begin(); l != deleted_leases->end(); ++l) {
1169 for (
auto l = leases->begin(); l != leases->end(); ++l) {
1191 for (
auto l = deleted_leases->begin(); l != deleted_leases->end(); ++l) {
1197 for (
auto l = leases->begin(); l != leases->end(); ++l) {
1222 size_t sent_num = 0;
1225 for (
auto p = peers_configs.begin(); p != peers_configs.end(); ++p) {
1232 for (
auto l = deleted_leases->begin(); l != deleted_leases->end(); ++l) {
1237 for (
auto l = leases->begin(); l != leases->end(); ++l) {
1273 template<
typename QueryPtrType>
1277 if (MultiThreadingMgr::instance().getMode()) {
1278 std::lock_guard<std::mutex> lock(mutex_);
1279 return (leaseUpdateCompleteInternal(query, parking_lot));
1281 return (leaseUpdateCompleteInternal(query, parking_lot));
1285 template<
typename QueryPtrType>
1287 HAService::leaseUpdateCompleteInternal(QueryPtrType& query,
1289 auto it = pending_requests_.find(query);
1293 if (it == pending_requests_.end() || (--pending_requests_[query] <= 0)) {
1294 parking_lot->unpark(query);
1298 if (it != pending_requests_.end()) {
1299 pending_requests_.erase(it);
1306 template<
typename QueryPtrType>
1309 if (MultiThreadingMgr::instance().getMode()) {
1310 std::lock_guard<std::mutex> lock(mutex_);
1311 updatePendingRequestInternal(query);
1313 updatePendingRequestInternal(query);
1317 template<
typename QueryPtrType>
1319 HAService::updatePendingRequestInternal(QueryPtrType& query) {
1320 if (pending_requests_.count(query) == 0) {
1321 pending_requests_[query] = 1;
1323 ++pending_requests_[query];
1327 template<
typename QueryPtrType>
1335 (HttpRequest::Method::HTTP_POST,
"/", HttpVersion::HTTP_11(),
1337 config->addBasicAuthHttpHeader(request);
1338 request->setBodyAsJson(command);
1339 request->finalize();
1348 boost::weak_ptr<typename QueryPtrType::element_type> weak_query(query);
1351 client_->asyncSendRequest(config->getUrl(), config->getTlsContext(),
1353 [
this, weak_query, parking_lot, config]
1354 (
const boost::system::error_code& ec,
1356 const std::string& error_str) {
1360 QueryPtrType query = weak_query.lock();
1363 " HA peer. This is programmatic error");
1372 bool lease_update_success =
true;
1375 if (ec || !error_str.empty()) {
1377 .arg(query->getLabel())
1378 .arg(config->getLogLabel())
1379 .arg(ec ? ec.message() : error_str);
1383 lease_update_success =
false;
1395 }
catch (
const std::exception& ex) {
1397 .arg(query->getLabel())
1398 .arg(config->getLogLabel())
1402 lease_update_success =
false;
1421 if (!lease_update_success) {
1422 parking_lot->drop(query);
1458 if (!
config_->amSendingLeaseUpdates()) {
1489 if (!
config_->amSendingLeaseUpdates()) {
1504 if (!args || (args->getType() != Element::map)) {
1514 auto failed_leases = args->get(param_name);
1517 if (failed_leases && (failed_leases->getType() == Element::list)) {
1519 for (
int i = 0; i < failed_leases->size(); ++i) {
1520 auto lease = failed_leases->get(i);
1521 if (lease->getType() == Element::map) {
1524 auto ip_address = lease->get(
"ip-address");
1527 auto lease_type = lease->get(
"type");
1530 auto error_message = lease->get(
"error-message");
1533 .arg(query->getLabel())
1534 .arg(lease_type && (lease_type->getType() == Element::string) ?
1535 lease_type->stringValue() :
"(unknown)")
1536 .arg(ip_address && (ip_address->getType() == Element::string) ?
1537 ip_address->stringValue() :
"(unknown)")
1538 .arg(error_message && (error_message->getType() == Element::string) ?
1539 error_message->stringValue() :
"(unknown)");
1554 ElementPtr ha_servers = Element::createMap();
1559 role =
config_->getThisServerConfig()->getRole();
1561 local->set(
"role", Element::create(role_txt));
1568 local->set(
"state", Element::create(std::string()));
1572 for (std::string scope : scopes) {
1573 list->add(Element::create(scope));
1575 local->set(
"scopes", list);
1576 ha_servers->set(
"local", local);
1582 return (ha_servers);
1589 role =
config_->getFailoverPeerConfig()->getRole();
1591 remote->set(
"role", Element::create(role_txt));
1594 remote->set(
"role", Element::create(std::string()));
1596 ha_servers->set(
"remote", remote);
1598 return (ha_servers);
1605 arguments->set(
"state", Element::create(state_label));
1608 arguments->set(
"date-time", Element::create(date_time));
1611 ElementPtr scopes_list = Element::createList();
1612 for (
auto scope : scopes) {
1613 scopes_list->add(Element::create(scope));
1615 arguments->set(
"scopes", scopes_list);
1617 arguments->set(
"unsent-update-count",
1650 (HttpRequest::Method::HTTP_POST,
"/", HttpVersion::HTTP_11(),
1652 partner_config->addBasicAuthHttpHeader(request);
1654 request->finalize();
1661 client_->asyncSendRequest(partner_config->getUrl(),
1662 partner_config->getTlsContext(),
1664 [
this, partner_config, sync_complete_notified]
1665 (
const boost::system::error_code& ec,
1667 const std::string& error_str) {
1675 bool heartbeat_success =
true;
1678 if (ec || !error_str.empty()) {
1680 .arg(partner_config->getLogLabel())
1681 .arg(ec ? ec.message() : error_str);
1682 heartbeat_success =
false;
1692 if (!args || args->getType() != Element::map) {
1698 if (!state || state->getType() != Element::string) {
1700 " to a ha-heartbeat command or it is not a string");
1707 if (!date_time || date_time->getType() != Element::string) {
1709 " to a ha-heartbeat command or it is not a string");
1716 auto scopes = args->get(
"scopes");
1732 auto unsent_update_count = args->get(
"unsent-update-count");
1733 if (unsent_update_count) {
1734 if (unsent_update_count->getType() != Element::integer) {
1736 " the ha-heartbeat response is not an integer");
1739 (unsent_update_count->intValue()));
1742 }
catch (
const std::exception& ex) {
1744 .arg(partner_config->getLogLabel())
1746 heartbeat_success =
false;
1752 if (heartbeat_success) {
1762 .arg(partner_config->getName());
1770 if (sync_complete_notified && !heartbeat_success) {
1796 if (
config_->getHeartbeatDelay() > 0) {
1805 const std::string& server_name,
1806 const unsigned int max_period,
1812 (HttpRequest::Method::HTTP_POST,
"/", HttpVersion::HTTP_11(),
1815 remote_config->addBasicAuthHttpHeader(request);
1818 request->finalize();
1826 remote_config->getTlsContext(),
1828 [
this, remote_config, post_request_action]
1829 (
const boost::system::error_code& ec,
1831 const std::string& error_str) {
1840 std::string error_message;
1843 if (ec || !error_str.empty()) {
1844 error_message = (ec ? ec.message() : error_str);
1846 .arg(remote_config->getLogLabel())
1847 .arg(error_message);
1855 }
catch (
const std::exception& ex) {
1856 error_message = ex.what();
1858 .arg(remote_config->getLogLabel())
1859 .arg(error_message);
1865 if (!error_message.empty()) {
1870 if (post_request_action) {
1871 post_request_action(error_message.empty(),
1885 const std::string& server_name,
1891 (HttpRequest::Method::HTTP_POST,
"/", HttpVersion::HTTP_11(),
1893 remote_config->addBasicAuthHttpHeader(request);
1895 request->finalize();
1903 remote_config->getTlsContext(),
1905 [
this, remote_config, post_request_action]
1906 (
const boost::system::error_code& ec,
1908 const std::string& error_str) {
1917 std::string error_message;
1920 if (ec || !error_str.empty()) {
1921 error_message = (ec ? ec.message() : error_str);
1923 .arg(remote_config->getLogLabel())
1924 .arg(error_message);
1932 }
catch (
const std::exception& ex) {
1933 error_message = ex.what();
1935 .arg(remote_config->getLogLabel())
1936 .arg(error_message);
1942 if (!error_message.empty()) {
1947 if (post_request_action) {
1948 post_request_action(error_message.empty(),
1962 network_state_->disableService(NetworkState::Origin::HA_COMMAND);
1975 unsigned int dhcp_disable_timeout =
1976 static_cast<unsigned int>(
config_->getSyncTimeout() / 1000);
1977 if (dhcp_disable_timeout == 0) {
1979 dhcp_disable_timeout = 1;
1983 dhcp_disable_timeout,
LeasePtr(), null_action);
1988 const std::string& server_name,
1989 const unsigned int max_period,
1992 const bool dhcp_disabled) {
1999 [
this, &http_client, server_name, max_period, last_lease,
2000 post_sync_action, dhcp_disabled]
2001 (
const bool success,
const std::string& error_message,
const int) {
2009 last_lease, post_sync_action,
true);
2012 post_sync_action(success, error_message, dhcp_disabled);
2019 const std::string& server_name,
2020 const unsigned int max_period,
2023 const bool dhcp_disabled) {
2029 (HttpRequest::Method::HTTP_POST,
"/", HttpVersion::HTTP_11(),
2031 partner_config->addBasicAuthHttpHeader(request);
2034 boost::dynamic_pointer_cast<Lease4>(last_lease),
config_->getSyncPageLimit()));
2038 boost::dynamic_pointer_cast<Lease6>(last_lease),
config_->getSyncPageLimit()));
2040 request->finalize();
2048 partner_config->getTlsContext(),
2050 [
this, partner_config, post_sync_action, &http_client, server_name,
2051 max_period, dhcp_disabled]
2052 (
const boost::system::error_code& ec,
2054 const std::string& error_str) {
2066 std::string error_message;
2069 if (ec || !error_str.empty()) {
2070 error_message = (ec ? ec.message() : error_str);
2072 .arg(partner_config->getLogLabel())
2073 .arg(error_message);
2082 if (args && (args->getType() != Element::map)) {
2084 "arguments in the received response must be a map");
2088 if (!leases || (leases->getType() != Element::list)) {
2090 "server response does not contain leases argument or this" 2091 " argument is not a list");
2095 const auto& leases_element = leases->listValue();
2098 .arg(leases_element.size())
2101 for (
auto l = leases_element.begin(); l != leases_element.end(); ++l) {
2105 Lease4Ptr lease = Lease4::fromElement(*l);
2108 Lease4Ptr existing_lease = LeaseMgrFactory::instance().getLease4(lease->addr_);
2109 if (!existing_lease) {
2111 LeaseMgrFactory::instance().addLease(lease);
2113 }
else if (existing_lease->cltt_ < lease->cltt_) {
2119 Lease::syncCurrentExpirationTime(*existing_lease, *lease);
2120 LeaseMgrFactory::instance().updateLease4(lease);
2124 .arg(lease->addr_.toText())
2125 .arg(lease->subnet_id_);
2131 if ((leases_element.size() >=
config_->getSyncPageLimit()) &&
2132 (l + 1 == leases_element.end())) {
2133 last_lease = boost::dynamic_pointer_cast<
Lease>(lease);
2137 Lease6Ptr lease = Lease6::fromElement(*l);
2140 Lease6Ptr existing_lease = LeaseMgrFactory::instance().getLease6(lease->type_,
2142 if (!existing_lease) {
2144 LeaseMgrFactory::instance().addLease(lease);
2146 }
else if (existing_lease->cltt_ < lease->cltt_) {
2152 Lease::syncCurrentExpirationTime(*existing_lease, *lease);
2153 LeaseMgrFactory::instance().updateLease6(lease);
2157 .arg(lease->addr_.toText())
2158 .arg(lease->subnet_id_);
2164 if ((leases_element.size() >=
config_->getSyncPageLimit()) &&
2165 (l + 1 == leases_element.end())) {
2166 last_lease = boost::dynamic_pointer_cast<
Lease>(lease);
2170 }
catch (
const std::exception& ex) {
2177 }
catch (
const std::exception& ex) {
2178 error_message = ex.what();
2180 .arg(partner_config->getLogLabel())
2181 .arg(error_message);
2187 if (!error_message.empty()) {
2190 }
else if (last_lease) {
2194 post_sync_action, dhcp_disabled);
2199 if (post_sync_action) {
2200 post_sync_action(error_message.empty(),
2215 const unsigned int max_period) {
2216 std::string answer_message;
2217 int sync_status =
synchronize(answer_message, server_name, max_period);
2223 const unsigned int max_period) {
2228 [&](
const bool success,
const std::string& error_message,
2229 const bool dhcp_disabled) {
2234 status_message = error_message;
2240 if (dhcp_disabled) {
2246 [&](
const bool success,
2247 const std::string& error_message,
2254 [&](
const bool success,
2255 const std::string& error_message,
2260 if (!success && status_message.empty()) {
2261 status_message = error_message;
2273 if (!success && status_message.empty()) {
2274 status_message = error_message;
2287 [&](
const bool success,
2288 const std::string& error_message,
2290 if (!success && status_message.empty()) {
2291 status_message = error_message;
2323 if (!status_message.empty()) {
2328 .arg(status_message);
2335 status_message =
"Lease database synchronization complete.";
2370 (HttpRequest::Method::HTTP_POST,
"/", HttpVersion::HTTP_11(),
2372 config->addBasicAuthHttpHeader(request);
2373 request->setBodyAsJson(command);
2374 request->finalize();
2382 [
this, &http_client, config, post_request_action]
2383 (
const boost::system::error_code& ec,
2385 const std::string& error_str) {
2388 std::string error_message;
2390 if (ec || !error_str.empty()) {
2391 error_message = (ec ? ec.message() : error_str);
2393 .arg(config->getLogLabel())
2394 .arg(ec ? ec.message() : error_str);
2400 }
catch (
const std::exception& ex) {
2401 error_message = ex.what();
2403 .arg(config->getLogLabel())
2413 if (error_message.empty()) {
2416 post_request_action(error_message.empty(), error_message, rcode);
2424 if (num_updates == 0) {
2431 auto remote_config =
config_->getFailoverPeerConfig();
2432 bool updates_successful =
true;
2436 .arg(remote_config->getName());
2439 [&](
const bool success,
const std::string&,
const int) {
2441 updates_successful = success;
2453 if (updates_successful) {
2455 .arg(remote_config->getName())
2456 .arg(stopwatch.logFormatLastDuration());
2459 return (updates_successful);
2470 (HttpRequest::Method::HTTP_POST,
"/", HttpVersion::HTTP_11(),
2472 config->addBasicAuthHttpHeader(request);
2473 request->setBodyAsJson(command);
2474 request->finalize();
2482 [
this, config, post_request_action]
2483 (
const boost::system::error_code& ec,
2485 const std::string& error_str) {
2488 std::string error_message;
2490 if (ec || !error_str.empty()) {
2491 error_message = (ec ? ec.message() : error_str);
2493 .arg(config->getLogLabel())
2494 .arg(ec ? ec.message() : error_str);
2500 }
catch (
const std::exception& ex) {
2501 error_message = ex.what();
2503 .arg(config->getLogLabel())
2508 post_request_action(error_message.empty(), error_message, rcode);
2516 auto remote_config =
config_->getFailoverPeerConfig();
2517 bool reset_successful =
true;
2520 [&](
const bool success,
const std::string&,
const int) {
2522 reset_successful = success;
2528 return (reset_successful);
2537 }
catch (
const std::exception& ex) {
2557 " maintenance for the server not in the" 2558 " in-maintenance state."));
2578 "Unable to transition the server from the " 2580 " in-maintenance state."));
2597 " partner-in-maintenance state."));
2607 (HttpRequest::Method::HTTP_POST,
"/", HttpVersion::HTTP_11(),
2609 remote_config->addBasicAuthHttpHeader(request);
2611 request->finalize();
2620 boost::system::error_code captured_ec;
2621 std::string captured_error_message;
2622 int captured_rcode = 0;
2626 remote_config->getTlsContext(),
2628 [
this, remote_config, &io_service, &captured_ec, &captured_error_message,
2630 (
const boost::system::error_code& ec,
2632 const std::string& error_str) {
2642 std::string error_message;
2645 if (ec || !error_str.empty()) {
2646 error_message = (ec ? ec.message() : error_str);
2648 .arg(remote_config->getLogLabel())
2649 .arg(error_message);
2657 }
catch (
const std::exception& ex) {
2658 error_message = ex.what();
2660 .arg(remote_config->getLogLabel())
2661 .arg(error_message);
2667 if (!error_message.empty()) {
2672 captured_error_message = error_message;
2691 "Server is now in the partner-down state as its" 2692 " partner appears to be offline for maintenance."));
2706 " partner-in-maintenance state. The partner server responded" 2707 " with the following message to the ha-maintenance-notify" 2708 " command: " + captured_error_message +
"."));
2713 "Server is now in the partner-in-maintenance state" 2714 " and its partner is in-maintenance state. The partner" 2715 " can be now safely shut down."));
2722 " request because the server is not in the" 2723 " partner-in-maintenance state."));
2731 (HttpRequest::Method::HTTP_POST,
"/", HttpVersion::HTTP_11(),
2733 remote_config->addBasicAuthHttpHeader(request);
2735 request->finalize();
2744 std::string error_message;
2748 remote_config->getTlsContext(),
2750 [
this, remote_config, &io_service, &error_message]
2751 (
const boost::system::error_code& ec,
2753 const std::string& error_str) {
2758 if (ec || !error_str.empty()) {
2759 error_message = (ec ? ec.message() : error_str);
2761 .arg(remote_config->getLogLabel())
2762 .arg(error_message);
2771 }
catch (
const std::exception& ex) {
2772 error_message = ex.what();
2774 .arg(remote_config->getLogLabel())
2775 .arg(error_message);
2781 if (!error_message.empty()) {
2797 if (!error_message.empty()) {
2799 "Unable to cancel maintenance. The partner server responded" 2800 " with the following message to the ha-maintenance-notify" 2801 " command: " + error_message +
"."));
2811 "Server maintenance successfully canceled."));
2816 const std::string& server_name,
2822 (HttpRequest::Method::HTTP_POST,
"/", HttpVersion::HTTP_11(),
2825 remote_config->addBasicAuthHttpHeader(request);
2827 request->finalize();
2835 remote_config->getTlsContext(),
2837 [
this, remote_config, post_request_action]
2838 (
const boost::system::error_code& ec,
2840 const std::string& error_str) {
2849 std::string error_message;
2852 if (ec || !error_str.empty()) {
2853 error_message = (ec ? ec.message() : error_str);
2855 .arg(remote_config->getLogLabel())
2856 .arg(error_message);
2864 }
catch (
const CommandUnsupportedError& ex) {
2867 }
catch (
const std::exception& ex) {
2868 error_message = ex.what();
2870 .arg(remote_config->getLogLabel())
2871 .arg(error_message);
2877 if (!error_message.empty()) {
2882 if (post_request_action) {
2883 post_request_action(error_message.empty(),
2903 "Server successfully notified about the synchronization completion."));
2913 if (!json_response) {
2924 if (body->getType() != Element::list) {
2926 if (body->getType() == Element::map) {
2942 if (body->empty()) {
2951 std::ostringstream s;
2953 if (args && args->getType() == Element::string) {
2954 s << args->stringValue() <<
", ";
2957 s <<
"error code " << rcode;
2960 isc_throw(CommandUnsupportedError, s.str());
2974 if (
client_->getThreadIOService()) {
2982 if ((!ec || (ec.value() == boost::asio::error::in_progress))
2983 && (tcp_native_fd >= 0)) {
2988 IfaceMgr::instance().addExternalSocket(tcp_native_fd,
3006 client_->closeIfOutOfBand(tcp_native_fd);
3011 if (tcp_native_fd >= 0) {
3012 IfaceMgr::instance().deleteExternalSocket(tcp_native_fd);
3018 if (MultiThreadingMgr::instance().getMode()) {
3019 std::lock_guard<std::mutex> lock(mutex_);
3020 return (pending_requests_.size());
3022 return (pending_requests_.size());
3026 template<
typename QueryPtrType>
3029 if (MultiThreadingMgr::instance().getMode()) {
3030 std::lock_guard<std::mutex> lock(mutex_);
3031 return (getPendingRequestInternal(query));
3033 return (getPendingRequestInternal(query));
3037 template<
typename QueryPtrType>
3039 HAService::getPendingRequestInternal(
const QueryPtrType& query) {
3040 if (pending_requests_.count(query) == 0) {
3043 return (pending_requests_[query]);
3068 }
catch (
const std::exception& ex) {
3077 MultiThreadingMgr::instance().addCriticalSectionCallbacks(
"HA_MT",
3103 }
catch (
const std::exception& ex) {
3121 }
catch (std::exception& ex) {
3130 MultiThreadingMgr::instance().removeCriticalSectionCallbacks(
"HA_MT");
void defineState(unsigned int value, const std::string &label, StateHandler handler, const StatePausing &state_pausing=STATE_PAUSE_NEVER)
Adds an state value and associated label to the set of states.
config::CmdHttpListenerPtr listener_
HTTP listener instance used to receive and respond to HA commands and lease updates.
Exception thrown when a worker thread is trying to stop or pause the respective thread pool (which wo...
static const int NOP_EVT
Signifies that no event has occurred.
static data::ConstElementPtr createHAReset(const HAServerType &server_type)
Creates ha-reset command.
const isc::log::MessageID HA_SYNC_FAILED
const int HA_TERMINATED_ST
HA service terminated state.
static const int HA_CONTROL_RESULT_MAINTENANCE_NOT_ALLOWED
Control result returned in response to ha-maintenance-notify.
#define LOG_WARN(LOGGER, MESSAGE)
Macro to conveniently test warn output and log it.
virtual void defineStates()
Defines states of the HA service.
const isc::log::MessageID HA_MAINTENANCE_NOTIFY_COMMUNICATIONS_FAILED
void serveDefaultScopes()
Instructs the HA service to serve default scopes.
data::ConstElementPtr processHAReset()
Processes ha-reset command and returns a response.
const int HA_COMMUNICATION_RECOVERY_ST
Communication recovery state.
void pauseClientAndListener()
Pauses client and(or) listener thread pool operations.
const isc::log::MessageID HA_STATE_TRANSITION
const int HA_HOT_STANDBY_ST
Hot standby state.
const char * CONTROL_RESULT
String used for result, i.e. integer status ("result")
Structure that holds a lease for IPv4 address.
bool leaseUpdateComplete(QueryPtrType &query, const hooks::ParkingLotHandlePtr &parking_lot)
Handle last pending request for this query.
const isc::log::MessageID HA_LEASES_BACKLOG_COMMUNICATIONS_FAILED
static const int HA_MAINTENANCE_CANCEL_EVT
ha-maintenance-cancel command received.
void defineEvent(unsigned int value, const std::string &label)
Adds an event value and associated label to the set of events.
const int DBGLVL_TRACE_BASIC
Trace basic operations.
bool doOnExit()
Checks if on exit flag is true.
void readyStateHandler()
Handler for "ready" state.
void serveNoScopes()
Disables all scopes.
void run()
Start the underlying event loop.
#define LOG_INFO(LOGGER, MESSAGE)
Macro to conveniently test info output and log it.
ConstElementPtr createAnswer(const int status_code, const std::string &text, const ConstElementPtr &arg)
const StatePtr getState(unsigned int value)
Fetches the state referred to by value.
const int CONTROL_RESULT_SUCCESS
Status code indicating a successful operation.
void asyncSendRequest(const Url &url, const asiolink::TlsContextPtr &tls_context, const HttpRequestPtr &request, const HttpResponsePtr &response, const RequestHandler &request_callback, const RequestTimeout &request_timeout=RequestTimeout(10000), const ConnectHandler &connect_callback=ConnectHandler(), const HandshakeHandler &handshake_callback=HandshakeHandler(), const CloseHandler &close_callback=CloseHandler())
Queues new asynchronous HTTP request for a given URL.
size_t pendingRequestSize()
Get the number of entries in the pending request map.
data::ConstElementPtr processScopes(const std::vector< std::string > &scopes)
Processes ha-scopes command and returns a response.
void scheduleHeartbeat()
Schedules asynchronous heartbeat to a peer if it is not scheduled.
std::map< std::string, PeerConfigPtr > PeerConfigMap
Map of the servers' configurations.
void asyncEnableDHCPService(http::HttpClient &http_client, const std::string &server_name, PostRequestCallback post_request_action)
Schedules asynchronous "dhcp-enable" command to the specified server.
boost::shared_ptr< HttpResponseJson > HttpResponseJsonPtr
Pointer to the HttpResponseJson object.
const isc::log::MessageID HA_INVALID_PARTNER_STATE_COMMUNICATION_RECOVERY
const isc::log::MessageID HA_LEASE_UPDATES_ENABLED
bool unpause()
Unpauses the HA state machine with logging.
const isc::log::MessageID HA_LEASES_BACKLOG_NOTHING_TO_SEND
void passiveBackupStateHandler()
Handler for "passive-backup" state.
const int HA_PARTNER_DOWN_ST
Partner down state.
void asyncSyncLeases()
Asynchronously reads leases from a peer and updates local lease database.
const isc::log::MessageID HA_LEASE_UPDATE_DELETE_FAILED_ON_PEER
void asyncSyncCompleteNotify(http::HttpClient &http_client, const std::string &server_name, PostRequestCallback post_request_action)
Schedules asynchronous "ha-sync-complete-notify" command to the specified server. ...
data::ConstElementPtr processHeartbeat()
Processes ha-heartbeat command and returns a response.
QueryFilter query_filter_
Selects queries to be processed/dropped.
An abstract API for lease database.
LeaseUpdateBacklog lease_update_backlog_
Backlog of DHCP lease updates.
OpType
Type of the lease update (operation type).
const isc::log::MessageID HA_MAINTENANCE_STARTED_IN_PARTNER_DOWN
const isc::log::MessageID HA_LEASES_SYNC_COMMUNICATIONS_FAILED
CommunicationStatePtr communication_state_
Holds communication state with a peer.
static std::unordered_set< std::string > ha_commands6_
List of commands used by the High Availability in v6.
virtual void verifyEvents()
Verifies events used by the HA service.
const int HA_IN_MAINTENANCE_ST
In maintenance state.
HTTP request/response timeout value.
static const int HA_SYNCING_SUCCEEDED_EVT
Lease database synchronization succeeded.
bool clientConnectHandler(const boost::system::error_code &ec, int tcp_native_fd)
HttpClient connect callback handler.
const int HA_LOAD_BALANCING_ST
Load balancing state.
HAServerType server_type_
DHCP server type.
#define LOG_ERROR(LOGGER, MESSAGE)
Macro to conveniently test error output and log it.
boost::shared_ptr< IOService > IOServicePtr
Defines a smart pointer to an IOService instance.
const int CONTROL_RESULT_ERROR
Status code indicating a general failure.
boost::shared_ptr< Element > ElementPtr
const isc::log::MessageID HA_LEASE_SYNC_STALE_LEASE4_SKIP
static const int HA_MAINTENANCE_NOTIFY_EVT
ha-maintenance-notify command received.
const isc::log::MessageID HA_CONFIG_LEASE_UPDATES_DISABLED_REMINDER
const isc::log::MessageID HA_MAINTENANCE_NOTIFY_FAILED
const isc::log::MessageID HA_RESET_FAILED
data::ConstElementPtr processMaintenanceCancel()
Processes ha-maintenance-cancel command and returns a response.
const isc::log::MessageID HA_COMMUNICATION_INTERRUPTED
static data::ConstElementPtr createLease6GetPage(const dhcp::Lease6Ptr &lease6, const uint32_t limit)
Creates lease6-get-page command.
void waitingStateHandler()
Handler for "waiting" state.
const isc::log::MessageID HA_LEASES_BACKLOG_SUCCESS
void updatePendingRequest(QueryPtrType &query)
Update pending request counter for this query.
const int HA_PASSIVE_BACKUP_ST
In passive-backup state with a single active server and backup servers.
void partnerInMaintenanceStateHandler()
Handler for "partner-in-maintenance" state.
HAServerType
Lists possible server types for which HA service is created.
const int HA_BACKUP_ST
Backup state.
virtual ~HAService()
Destructor.
const isc::log::MessageID HA_LEASE_UPDATES_DISABLED
const isc::log::MessageID HA_RESET_COMMUNICATIONS_FAILED
bool shouldTerminate() const
Indicates if the server should transition to the terminated state as a result of high clock skew...
void logFailedLeaseUpdates(const dhcp::PktPtr &query, const data::ConstElementPtr &args) const
Log failed lease updates.
boost::shared_ptr< Lease > LeasePtr
Pointer to the lease object.
void serveDefaultScopes()
Serve default scopes for the given HA mode.
virtual const char * what() const
Returns a C-style character string of the cause of the exception.
const isc::log::MessageID HA_PAUSE_CLIENT_LISTENER_ILLEGAL
dhcp::NetworkStatePtr network_state_
Pointer to the state of the DHCP service (enabled/disabled).
static const int HA_MAINTENANCE_START_EVT
ha-maintenance-start command received.
unsigned int getLastEvent() const
Fetches the model's last event.
const isc::log::MessageID HA_RESUME_CLIENT_LISTENER_FAILED
const isc::log::MessageID HA_SYNC_COMPLETE_NOTIFY_FAILED
The IOService class is a wrapper for the ASIO io_service class.
virtual void runModel(unsigned int event)
Processes events through the state model.
const int CONTROL_RESULT_EMPTY
Status code indicating that the specified command was completed correctly, but failed to produce any ...
asiolink::IOServicePtr io_service_
Pointer to the IO service object shared between this hooks library and the DHCP server.
const int HA_WAITING_ST
Server waiting state, i.e. waiting for another server to be ready.
const isc::log::MessageID HA_INVALID_PARTNER_STATE_HOT_STANDBY
data::ConstElementPtr processContinue()
Processes ha-continue command and returns a response.
#define isc_throw(type, stream)
A shortcut macro to insert known values into exception arguments.
void stop()
Stops the stopwatch.
Holds communication state between DHCPv4 servers.
void resumeClientAndListener()
Resumes client and(or) listener thread pool operations.
boost::shared_ptr< Pkt6 > Pkt6Ptr
A pointer to Pkt6 packet.
void checkPermissionsClientAndListener()
Check client and(or) listener current thread permissions to perform thread pool state transition...
data::ConstElementPtr processSyncCompleteNotify()
Process ha-sync-complete-notify command and returns a response.
const int HA_PARTNER_IN_MAINTENANCE_ST
Partner in-maintenance state.
boost::shared_ptr< PostHttpRequestJson > PostHttpRequestJsonPtr
Pointer to PostHttpRequestJson.
const int HA_READY_ST
Server ready state, i.e. synchronized database, can enable DHCP service.
static const int HA_HEARTBEAT_COMPLETE_EVT
Finished heartbeat command.
const isc::log::MessageID HA_LEASE_UPDATE_CREATE_UPDATE_FAILED_ON_PEER
const isc::log::MessageID HA_DHCP_DISABLE_COMMUNICATIONS_FAILED
static const int HA_LEASE_UPDATES_COMPLETE_EVT
Finished lease updates commands.
boost::shared_ptr< Lease4 > Lease4Ptr
Pointer to a Lease4 structure.
static data::ConstElementPtr createLease4Update(const dhcp::Lease4 &lease4)
Creates lease4-update command.
void partnerDownStateHandler()
Handler for "partner-down" state.
void syncingStateHandler()
Handler for "syncing" state.
static const int HA_SYNCED_PARTNER_UNAVAILABLE_EVT
The heartbeat command failed after receiving ha-sync-complete-notify command from the partner...
void clear()
Removes all lease updates from the queue.
boost::shared_ptr< HttpResponse > HttpResponsePtr
Pointer to the HttpResponse object.
Utility class to measure code execution times.
void asyncSendHeartbeat()
Starts asynchronous heartbeat to a peer.
std::set< std::string > getServedScopes() const
Returns served scopes.
void serveScopes(const std::vector< std::string > &scopes)
Enables selected scopes.
const isc::log::MessageID HA_LEASES_BACKLOG_START
boost::shared_ptr< Lease4Collection > Lease4CollectionPtr
A shared pointer to the collection of IPv4 leases.
void inMaintenanceStateHandler()
Handler for the "in-maintenance" state.
A generic exception that is thrown when an unexpected error condition occurs.
void asyncSendLeaseUpdate(const QueryPtrType &query, const HAConfig::PeerConfigPtr &config, const data::ConstElementPtr &command, const hooks::ParkingLotHandlePtr &parking_lot)
Asynchronously sends lease update to the peer.
bool doOnEntry()
Checks if on entry flag is true.
const char * CONTROL_TEXT
String used for storing textual description ("text")
static data::ConstElementPtr createSyncCompleteNotify(const HAServerType &server_type)
Creates ha-sync-complete-notify command.
const isc::log::MessageID HA_LEASE_SYNC_FAILED
bool push(const OpType op_type, const dhcp::LeasePtr &lease)
Appends lease update to the queue.
const isc::log::MessageID HA_SERVICE_STARTED
std::string stateToString(int state)
Returns state name.
std::string rfc1123Format() const
Returns time value formatted as specified in RFC 1123.
const isc::log::MessageID HA_LOCAL_DHCP_ENABLE
boost::shared_ptr< Pkt4 > Pkt4Ptr
A pointer to Pkt4 object.
void stop()
Stop the underlying event loop.
boost::shared_ptr< const Element > ConstElementPtr
static std::unordered_set< std::string > ha_commands4_
List of commands used by the High Availability in v4.
std::string logFormatLastDuration() const
Returns the last measured duration in the format directly usable in log messages. ...
const isc::log::MessageID HA_STATE_TRANSITION_PASSIVE_BACKUP
const isc::log::MessageID HA_HEARTBEAT_FAILED
data::ConstElementPtr processMaintenanceNotify(const bool cancel)
Processes ha-maintenance-notify command and returns a response.
const isc::log::MessageID HA_DHCP_DISABLE_FAILED
size_t size()
Returns the current size of the queue.
unsigned int getNextEvent() const
Fetches the model's next event.
bool inScope(dhcp::Pkt4Ptr &query4)
Checks if the DHCPv4 query should be processed by this server.
void terminatedStateHandler()
Handler for "terminated" state.
const isc::log::MessageID HA_SYNC_COMPLETE_NOTIFY_COMMUNICATIONS_FAILED
bool wasOverflown()
Checks if the queue was overflown.
void stopClientAndListener()
Stop the client and(or) listener instances.
const isc::log::MessageID HA_LEASE_UPDATE_COMMUNICATIONS_FAILED
const isc::log::MessageID HA_LEASE_UPDATE_FAILED
void serveFailoverScopes()
Enable scopes required in failover case.
void localDisableDHCPService()
Disables local DHCP service.
boost::shared_ptr< isc::dhcp::Pkt > PktPtr
A pointer to either Pkt4 or Pkt6 packet.
unsigned int getCurrState() const
Fetches the model's current state.
std::function< void(const bool, const std::string &, const int)> PostRequestCallback
Callback invoked when request was sent and a response received or an error occurred.
void adjustNetworkState()
Enables or disables network state depending on the served scopes.
const isc::log::MessageID HA_INVALID_PARTNER_STATE_LOAD_BALANCING
This class parses and generates time values used in HTTP.
const isc::log::MessageID HA_SYNC_START
static data::ConstElementPtr createLease4Delete(const dhcp::Lease4 &lease4)
Creates lease4-del command.
void startModel(const int start_state)
Begins execution of the model.
void unpauseModel()
Unpauses state model.
Represents HTTP response with JSON content.
void asyncSyncLeasesInternal(http::HttpClient &http_client, const std::string &server_name, const unsigned int max_period, const dhcp::LeasePtr &last_lease, PostSyncCallback post_sync_action, const bool dhcp_disabled)
Implements fetching one page of leases during synchronization.
std::function< void(const bool, const std::string &, const bool)> PostSyncCallback
Callback invoked when lease database synchronization is complete.
void communicationRecoveryHandler()
Handler for the "communication-recovery" state.
A standard control channel exception that is thrown if a function is there is a problem with one of t...
ConstElementPtr parseAnswer(int &rcode, const ConstElementPtr &msg)
Defines the logger used by the top-level component of kea-lfc.
const isc::log::MessageID HA_STATE_MACHINE_PAUSED
bool shouldPartnerDown() const
Indicates if the server should transition to the partner down state.
const isc::log::MessageID HA_LOCAL_DHCP_DISABLE
const isc::log::MessageID HA_MAINTENANCE_NOTIFY_CANCEL_COMMUNICATIONS_FAILED
data::ConstElementPtr processMaintenanceStart()
Processes ha-maintenance-start command and returns a response.
bool isMaintenanceCanceled() const
Convenience method checking if the current state is a result of canceling the maintenance.
HAConfigPtr config_
Pointer to the HA hooks library configuration.
void startHeartbeat()
Unconditionally starts one heartbeat to a peer.
const isc::log::MessageID HA_MAINTENANCE_STARTED
data::ConstElementPtr processStatusGet() const
Processes status-get command and returns a response.
const isc::log::MessageID HA_HEARTBEAT_COMMUNICATIONS_FAILED
void transition(unsigned int state, unsigned int event)
Sets up the model to transition into given state with a given event.
bool sync_complete_notified_
An indicator that a partner sent ha-sync-complete-notify command.
void startClientAndListener()
Start the client and(or) listener instances.
http::HttpClientPtr client_
HTTP client instance used to send HA commands and lease updates.
const isc::log::MessageID HA_DHCP_ENABLE_FAILED
void normalStateHandler()
Handler for the "hot-standby" and "load-balancing" states.
This file contains several functions and constants that are used for handling commands and responses ...
bool sendLeaseUpdatesFromBacklog()
Attempts to send all lease updates from the backlog synchronously.
static std::string HAModeToString(const HAMode &ha_mode)
Returns HA mode name.
void localEnableDHCPService()
Enables local DHCP service.
a common structure for IPv4 and IPv6 leases
virtual void defineEvents()
Defines events used by the HA service.
std::string getStateLabel(const int state) const
Fetches the label associated with an state value.
boost::shared_ptr< Lease6Collection > Lease6CollectionPtr
A shared pointer to the collection of IPv6 leases.
boost::shared_ptr< NetworkState > NetworkStatePtr
Pointer to the NetworkState object.
unsigned int getPrevState() const
Fetches the model's previous state.
#define LOG_DEBUG(LOGGER, LEVEL, MESSAGE)
Macro to conveniently test debug output and log it.
static data::ConstElementPtr createLease6BulkApply(const dhcp::Lease6CollectionPtr &leases, const dhcp::Lease6CollectionPtr &deleted_leases)
Creates lease6-bulk-apply command.
isc::log::Logger ha_logger("ha-hooks")
bool isPartnerStateInvalid() const
Indicates if the partner's state is invalid.
bool shouldSendLeaseUpdates(const HAConfig::PeerConfigPtr &peer_config) const
Checks if the lease updates should be sent as result of leases allocation or release.
const isc::log::MessageID HA_LEASE_SYNC_STALE_LEASE6_SKIP
void backupStateHandler()
Handler for the "backup" state.
int getPendingRequest(const QueryPtrType &query)
Get the number of scheduled requests for a given query.
const isc::log::MessageID HA_SYNC_SUCCESSFUL
const isc::log::MessageID HA_LEASES_BACKLOG_FAILED
static const int HA_SYNCING_FAILED_EVT
Lease database synchronization failed.
bool inScope(const dhcp::Pkt4Ptr &query4, std::string &scope_class) const
Checks if this server should process the DHCPv4 query.
A multi-threaded HTTP listener that can process API commands requests.
Role
Server's role in the High Availability setup.
const isc::log::MessageID HA_MAINTENANCE_SHUTDOWN_SAFE
size_t asyncSendLeaseUpdates(const dhcp::Pkt4Ptr &query, const dhcp::Lease4CollectionPtr &leases, const dhcp::Lease4CollectionPtr &deleted_leases, const hooks::ParkingLotHandlePtr &parking_lot)
Schedules asynchronous IPv4 leases updates.
boost::shared_ptr< ParkingLotHandle > ParkingLotHandlePtr
Pointer to the parking lot handle.
void asyncSendHAReset(http::HttpClient &http_client, const HAConfig::PeerConfigPtr &remote_config, PostRequestCallback post_request_action)
Sends ha-reset command to partner asynchronously.
dhcp::LeasePtr pop(OpType &op_type)
Returns the next lease update and removes it from the queue.
const isc::log::MessageID HA_PAUSE_CLIENT_LISTENER_FAILED
const isc::log::MessageID HA_CONFIG_LEASE_SYNCING_DISABLED_REMINDER
const isc::log::MessageID HA_MAINTENANCE_NOTIFY_CANCEL_FAILED
static std::string roleToString(const HAConfig::PeerConfig::Role &role)
Returns role name.
const isc::log::MessageID HA_STATE_MACHINE_CONTINUED
void socketReadyHandler(int tcp_native_fd)
IfaceMgr external socket ready callback handler.
static data::ConstElementPtr createHeartbeat(const HAServerType &server_type)
Creates ha-heartbeat command for DHCP server.
std::string ClientClass
Defines a single class name.
int getNormalState() const
Returns normal operation state for the current configuration.
The IOAddress class represents an IP addresses (version agnostic)
const EventPtr & getEvent(unsigned int value)
Fetches the event referred to by value.
void conditionalLogPausedState() const
Logs if the server is paused in the current state.
const isc::log::MessageID HA_TERMINATED_RESTART_PARTNER
constexpr long TIMEOUT_DEFAULT_HTTP_CLIENT_REQUEST
Timeout for the HTTP clients awaiting a response to a request.
data::ConstElementPtr processSynchronize(const std::string &server_name, const unsigned int max_period)
Processes ha-sync command and returns a response.
bool isModelPaused() const
Returns whether or not the model is paused.
const int HA_SYNCING_ST
Synchronizing database state.
const isc::log::MessageID HA_DHCP_ENABLE_COMMUNICATIONS_FAILED
const isc::log::MessageID HA_LEASES_SYNC_LEASE_PAGE_RECEIVED
boost::shared_ptr< Lease6 > Lease6Ptr
Pointer to a Lease6 structure.
const isc::log::MessageID HA_LEASES_SYNC_FAILED
static data::ConstElementPtr createMaintenanceNotify(const bool cancel, const HAServerType &server_type)
Creates ha-maintenance-notify command.
data::ConstElementPtr verifyAsyncResponse(const http::HttpResponsePtr &response, int &rcode)
Checks if the response is valid or contains an error.
static data::ConstElementPtr createDHCPEnable(const HAServerType &server_type)
Creates dhcp-enable command for DHCP server.
void clientCloseHandler(int tcp_native_fd)
HttpClient close callback handler.
void verboseTransition(const unsigned state)
Transitions to a desired state and logs it.
bool shouldQueueLeaseUpdates(const HAConfig::PeerConfigPtr &peer_config) const
Checks if the lease updates should be queued.
boost::shared_ptr< HAConfig > HAConfigPtr
Pointer to the High Availability configuration structure.
bool sendHAReset()
Sends ha-reset command to partner synchronously.
static data::ConstElementPtr createDHCPDisable(const unsigned int max_period, const HAServerType &server_type)
Creates dhcp-disable command for DHCP server.
Holds communication state between DHCPv6 servers.
const int HA_UNAVAILABLE_ST
Special state indicating that this server is unable to communicate with the partner.
void postNextEvent(unsigned int event)
Sets the next event to the given event value.
int synchronize(std::string &status_message, const std::string &server_name, const unsigned int max_period)
Synchronizes lease database with a partner.
bool clientHandshakeHandler(const boost::system::error_code &)
HttpClient handshake callback handler.
boost::shared_ptr< PeerConfig > PeerConfigPtr
Pointer to the server's configuration.
const int CONTROL_RESULT_COMMAND_UNSUPPORTED
Status code indicating that the specified command is not supported.
static data::ConstElementPtr createLease4GetPage(const dhcp::Lease4Ptr &lease4, const uint32_t limit)
Creates lease4-get-page command.
const isc::log::MessageID HA_TERMINATED
void asyncDisableDHCPService(http::HttpClient &http_client, const std::string &server_name, const unsigned int max_period, PostRequestCallback post_request_action)
Schedules asynchronous "dhcp-disable" command to the specified server.
void asyncSendLeaseUpdatesFromBacklog(http::HttpClient &http_client, const HAConfig::PeerConfigPtr &remote_config, PostRequestCallback post_request_action)
Sends lease updates from backlog to partner asynchronously.