Kea 3.3.4
ha_service.cc
Go to the documentation of this file.
1// Copyright (C) 2018-2026 Internet Systems Consortium, Inc. ("ISC")
2//
3// This Source Code Form is subject to the terms of the Mozilla Public
4// License, v. 2.0. If a copy of the MPL was not distributed with this
5// file, You can obtain one at http://mozilla.org/MPL/2.0/.
6
7#include <config.h>
8
10#include <cc/data.h>
12#include <config/timeouts.h>
13#include <dhcp/iface_mgr.h>
14#include <dhcpsrv/cfgmgr.h>
15#include <dhcpsrv/lease_mgr.h>
21#include <http/date_time.h>
23#include <http/response_json.h>
26#include <util/stopwatch.h>
27
28#include <functional>
29#include <sstream>
30
31#include <boost/make_shared.hpp>
32#include <boost/pointer_cast.hpp>
33#include <boost/weak_ptr.hpp>
34
35#include <ha_log.h>
36
37using namespace isc::asiolink;
38using namespace isc::config;
39using namespace isc::data;
40using namespace isc::dhcp;
41using namespace isc::hooks;
42using namespace isc::http;
43using namespace isc::log;
44using namespace isc::util;
45namespace ph = std::placeholders;
46
47namespace {
48
50class CommandUnsupportedError : public CtrlChannelError {
51public:
52 CommandUnsupportedError(const char* file, size_t line, const char* what) :
53 CtrlChannelError(file, line, what) {}
54};
55
57class ConflictError : public CtrlChannelError {
58public:
59 ConflictError(const char* file, size_t line, const char* what) :
60 CtrlChannelError(file, line, what) {}
61};
62
63}
64
65namespace isc {
66namespace ha {
67
76
79
80HAService::HAService(const unsigned int id, const IOServicePtr& io_service,
81 const NetworkStatePtr& network_state, const HAConfigPtr& config,
82 const HAServerType& server_type)
83 : id_(id), io_service_(io_service), network_state_(network_state), config_(config),
85 query_filter_(config), lease_sync_filter_(server_type, config), mutex_(),
86 pending_requests_(), lease_update_backlog_(config->getDelayedUpdatesLimit()),
88
89 if (server_type == HAServerType::DHCPv4) {
91
92 } else {
94 }
95
96 network_state_->enableService(getLocalOrigin());
97
99
100 // Create the client and(or) listener as appropriate.
101 if (!config_->getEnableMultiThreading()) {
102 // Not configured for multi-threading, start a client in ST mode.
103 client_.reset(new HttpClient(io_service_, false));
104 } else {
105 // Create an MT-mode client.
106 client_.reset(new HttpClient(io_service_, true,
107 config_->getHttpClientThreads(), true));
108
109 // If we're configured to use our own listener create and start it.
110 if (config_->getHttpDedicatedListener()) {
111 // Get the server address and port from this server's URL.
112 auto my_url = config_->getThisServerConfig()->getUrl();
113 IOAddress server_address(IOAddress::IPV4_ZERO_ADDRESS());
114 try {
115 // Since we do not currently support hostname resolution,
116 // we need to make sure we have an IP address here.
117 server_address = IOAddress(my_url.getStrippedHostname());
118 } catch (const std::exception& ex) {
119 isc_throw(Unexpected, "server Url:" << my_url.getStrippedHostname()
120 << " is not a valid IP address");
121 }
122
123 // Fetch how many threads the listener will use.
124 uint32_t listener_threads = config_->getHttpListenerThreads();
125
126 // Fetch the TLS context.
127 auto tls_context = config_->getThisServerConfig()->getTlsContext();
128
129 // Set the HTTP basic authentication.
130 HttpAuthConfigPtr auth_config =
131 config_->getThisServerConfig()->getBasicAuthConfig();
132
133 // Set the command filter when enabled.
134 std::unordered_set<std::string> command_accept_list;
135 if (config_->getRestrictCommands()) {
136 if (server_type == HAServerType::DHCPv4) {
137 command_accept_list = CommandCreator::ha_commands4_;
138 } else {
139 command_accept_list = CommandCreator::ha_commands6_;
140 }
141 }
142
143 // Instantiate the listener.
144 listener_.reset(new CmdHttpListener(server_address,
145 my_url.getPort(),
146 listener_threads,
147 tls_context,
148 auth_config,
149 command_accept_list));
150 }
151 }
152
154 .arg(config_->getThisServerName())
155 .arg(HAConfig::HAModeToString(config->getHAMode()))
156 .arg(HAConfig::PeerConfig::roleToString(config->getThisServerConfig()->getRole()));
157}
158
160 // Stop client and/or listener.
162
163 network_state_->enableService(getLocalOrigin());
164}
165
166std::string
167HAService::getCSCallbacksSetName() const {
168 std::ostringstream s;
169 s << "HA_MT_" << id_;
170 return (s.str());
171}
172
173void
176
177 defineEvent(HA_HEARTBEAT_COMPLETE_EVT, "HA_HEARTBEAT_COMPLETE_EVT");
178 defineEvent(HA_LEASE_UPDATES_COMPLETE_EVT, "HA_LEASE_UPDATES_COMPLETE_EVT");
179 defineEvent(HA_SYNCING_FAILED_EVT, "HA_SYNCING_FAILED_EVT");
180 defineEvent(HA_SYNCING_SUCCEEDED_EVT, "HA_SYNCING_SUCCEEDED_EVT");
181 defineEvent(HA_MAINTENANCE_NOTIFY_EVT, "HA_MAINTENANCE_NOTIFY_EVT");
182 defineEvent(HA_MAINTENANCE_START_EVT, "HA_MAINTENANCE_START_EVT");
183 defineEvent(HA_MAINTENANCE_CANCEL_EVT, "HA_MAINTENANCE_CANCEL_EVT");
184 defineEvent(HA_SYNCED_PARTNER_UNAVAILABLE_EVT, "HA_SYNCED_PARTNER_UNAVAILABLE_EVT");
185}
186
187void
190
199}
200
201void
204
206 std::bind(&HAService::backupStateHandler, this),
207 config_->getStateMachineConfig()->getStateConfig(HA_BACKUP_ST)->getPausing());
208
210 std::bind(&HAService::communicationRecoveryStateHandler, this),
211 config_->getStateMachineConfig()->getStateConfig(HA_COMMUNICATION_RECOVERY_ST)->getPausing());
212
214 std::bind(&HAService::normalStateHandler, this),
215 config_->getStateMachineConfig()->getStateConfig(HA_HOT_STANDBY_ST)->getPausing());
216
218 std::bind(&HAService::normalStateHandler, this),
219 config_->getStateMachineConfig()->getStateConfig(HA_LOAD_BALANCING_ST)->getPausing());
220
222 std::bind(&HAService::inMaintenanceStateHandler, this),
223 config_->getStateMachineConfig()->getStateConfig(HA_IN_MAINTENANCE_ST)->getPausing());
224
226 std::bind(&HAService::partnerDownStateHandler, this),
227 config_->getStateMachineConfig()->getStateConfig(HA_PARTNER_DOWN_ST)->getPausing());
228
230 std::bind(&HAService::partnerInMaintenanceStateHandler, this),
231 config_->getStateMachineConfig()->getStateConfig(HA_PARTNER_IN_MAINTENANCE_ST)->getPausing());
232
234 std::bind(&HAService::passiveBackupStateHandler, this),
235 config_->getStateMachineConfig()->getStateConfig(HA_PASSIVE_BACKUP_ST)->getPausing());
236
238 std::bind(&HAService::readyStateHandler, this),
239 config_->getStateMachineConfig()->getStateConfig(HA_READY_ST)->getPausing());
240
242 std::bind(&HAService::syncingStateHandler, this),
243 config_->getStateMachineConfig()->getStateConfig(HA_SYNCING_ST)->getPausing());
244
246 std::bind(&HAService::terminatedStateHandler, this),
247 config_->getStateMachineConfig()->getStateConfig(HA_TERMINATED_ST)->getPausing());
248
250 std::bind(&HAService::waitingStateHandler, this),
251 config_->getStateMachineConfig()->getStateConfig(HA_WAITING_ST)->getPausing());
252
254 std::bind(&HAService::downStateHandler, this),
255 config_->getStateMachineConfig()->getStateConfig(HA_DOWN_ST)->getPausing());
256}
257
258void
259HAService::backupStateHandler() {
260 // If we are transitioning from another state, we have to define new
261 // serving scopes appropriate for the new state. We don't do it if
262 // we remain in this state.
263 if (doOnEntry()) {
264 query_filter_.serveNoScopes();
266
267 // Log if the state machine is paused.
269 }
270
271 // There is nothing to do in that state. This server simply receives
272 // lease updates from the partners.
274}
275
276void
277HAService::communicationRecoveryStateHandler() {
278 if (doOnEntry()) {
279 query_filter_.serveDefaultScopes();
281
282 // Log if the state machine is paused.
284 }
285
287
290
291 // Check if the clock skew is still acceptable. If not, transition to
292 // the terminated state.
293 } else if (shouldTerminate()) {
295
296 } else if (isDHCPServiceDisabled()) {
298
299 } else if (isPartnerStateInvalid()) {
301
302 } else {
303
304 // Transitions based on the partner's state.
305 switch (communication_state_->getPartnerState()) {
308 break;
309
310 case HA_DOWN_ST:
312 break;
313
316 break;
317
320 break;
321
322 case HA_TERMINATED_ST:
324 break;
325
327 if (shouldPartnerDown()) {
329
330 } else {
332 }
333 break;
334
335 case HA_WAITING_ST:
336 case HA_SYNCING_ST:
337 case HA_READY_ST:
338 // The partner seems to be waking up, perhaps after communication-recovery.
339 // If our backlog queue is overflown we need to synchronize our lease database.
340 // There is no need to send ha-reset to the partner because the partner is
341 // already synchronizing its lease database.
342 if (!communication_state_->isCommunicationInterrupted() &&
343 lease_update_backlog_.wasOverflown()) {
345 } else {
346 // Backlog was not overflown, so there is no need to synchronize our
347 // lease database. Let's wait until our partner completes synchronization
348 // and transitions to the load-balancing state.
350 }
351 break;
352
353 default:
354 // If the communication is still interrupted, let's continue sitting
355 // in this state until it is resumed or until the transition to the
356 // partner-down state, depending on what happens first.
357 if (communication_state_->isCommunicationInterrupted()) {
359 break;
360 }
361
362 // The communication has been resumed. The partner server must be in a state
363 // in which it can receive outstanding lease updates we collected. The number of
364 // outstanding lease updates must not exceed the configured limit. Finally, the
365 // lease updates must be successfully sent. If that all works, we will transition
366 // to the normal operation.
367 if ((communication_state_->getPartnerState() == getNormalState()) ||
368 (communication_state_->getPartnerState() == HA_COMMUNICATION_RECOVERY_ST)) {
369 if (lease_update_backlog_.wasOverflown() || !sendLeaseUpdatesFromBacklog()) {
370 // If our lease backlog was overflown or we were unable to send lease
371 // updates to the partner we should notify the partner that it should
372 // synchronize the lease database. We do it by sending ha-reset command.
373 if (sendHAReset()) {
375 }
376 break;
377 }
378 // The backlog was not overflown and we successfully sent our lease updates.
379 // We can now transition to the normal operation state. If the partner
380 // fails to send his outstanding lease updates to us it should send the
381 // ha-reset command to us.
383 break;
384 }
385
386 // The partner appears to be in unexpected state, we have exceeded the number
387 // of lease updates in a backlog or an attempt to send lease updates failed.
388 // In all these cases we follow plan B and transition to the waiting state.
389 // The server will then attempt to synchronize the entire lease database.
391 }
392 }
393
394 // When exiting this state we must ensure that lease updates backlog is cleared.
395 if (doOnExit()) {
396 lease_update_backlog_.clear();
397 }
398}
399
400void
401HAService::normalStateHandler() {
402 // If we are transitioning from another state, we have to define new
403 // serving scopes appropriate for the new state. We don't do it if
404 // we remain in this state.
405 if (doOnEntry()) {
406 query_filter_.serveDefaultScopes();
408
409 // Log if the state machine is paused.
411 }
412
414
417 return;
418 }
419
420 // Check if the clock skew is still acceptable. If not, transition to
421 // the terminated state.
422 if (shouldTerminate()) {
424 return;
425 }
426
427 if (isDHCPServiceDisabled()) {
429 return;
430 }
431
432 // Check if the partner state is valid per current configuration. If it is
433 // in an invalid state let's transition to the waiting state and stay there
434 // until the configuration is corrected.
435 if (isPartnerStateInvalid()) {
437 return;
438 }
439
440 switch (communication_state_->getPartnerState()) {
443 break;
444
447 break;
448
449 case HA_DOWN_ST:
451 break;
452
455 break;
456
457 case HA_TERMINATED_ST:
459 break;
460
462 if (shouldPartnerDown()) {
464
465 } else if (config_->amAllowingCommRecovery()) {
467
468 } else {
470 }
471 break;
472
473 default:
475 }
476
477 if (doOnExit()) {
478 // Do nothing here but doOnExit() call clears the "on exit" flag
479 // when transitioning to the communication-recovery state. In that
480 // state we need this flag to be cleared.
481 }
482}
483
484void
485HAService::inMaintenanceStateHandler() {
486 // If we are transitioning from another state, we have to define new
487 // serving scopes appropriate for the new state. We don't do it if
488 // we remain in this state.
489 if (doOnEntry()) {
490 // In this state the server remains silent and waits for being
491 // shutdown.
492 query_filter_.serveNoScopes();
494
495 // Log if the state machine is paused.
497
499 .arg(config_->getThisServerName());
500 }
501
503
504 // We don't transition out of this state unless explicitly mandated
505 // by the administrator via a dedicated command which cancels
506 // the maintenance.
508}
509
510void
511HAService::downStateHandler() {
512 // If we are transitioning from another state, we have to define new
513 // serving scopes appropriate for the new state. We don't do it if
514 // we remain in this state.
515 if (doOnEntry()) {
516 query_filter_.serveDefaultScopes();
518
519 // Log if the state machine is paused.
521 }
522
524
527 return;
528 }
529
530 // Check if the clock skew is still acceptable. If not, transition to
531 // the terminated state.
532 if (shouldTerminate()) {
534 return;
535 }
536
537 switch (communication_state_->getPartnerState()) {
538 case HA_TERMINATED_ST:
540 return;
541 default:
543 }
544
545 if (!isDHCPServiceDisabled()) {
548 }
549
551}
552
553void
554HAService::partnerDownStateHandler() {
555 // If we are transitioning from another state, we have to define new
556 // serving scopes appropriate for the new state. We don't do it if
557 // we remain in this state.
558 if (doOnEntry()) {
559
560 bool maintenance = (getLastEvent() == HA_MAINTENANCE_START_EVT);
561
562 // It may be administratively disabled to handle partner's scope
563 // in case of failure. If this is the case we'll just handle our
564 // default scope (or no scope at all). The user will need to
565 // manually enable this server to handle partner's scope.
566 // If we're in the maintenance mode we serve all scopes because
567 // it is not a failover situation.
568 if (maintenance || config_->getThisServerConfig()->isAutoFailover()) {
569 query_filter_.serveFailoverScopes();
570 } else {
571 query_filter_.serveDefaultScopes();
572 }
574 communication_state_->clearRejectedLeaseUpdates();
575
576 // Log if the state machine is paused.
578
579 if (maintenance) {
580 // If we ended up in the partner-down state as a result of
581 // receiving the ha-maintenance-start command let's log it.
583 .arg(config_->getThisServerName());
584 }
585
587 // Partner sent the ha-sync-complete-notify command to indicate that
588 // it has successfully synchronized its lease database but that server
589 // was unable to send heartbeat to this server. Enable the DHCP service
590 // and continue serving the clients in the partner-down state until the
591 // communication with the partner is fixed.
593 }
594
596
599 return;
600 }
601
602 // Check if the clock skew is still acceptable. If not, transition to
603 // the terminated state.
604 if (shouldTerminate()) {
606 return;
607 }
608
609 if (isDHCPServiceDisabled()) {
611 return;
612 }
613
614 // Check if the partner state is valid per current configuration. If it is
615 // in an invalid state let's transition to the waiting state and stay there
616 // until the configuration is corrected.
617 if (isPartnerStateInvalid()) {
619 return;
620 }
621
622 switch (communication_state_->getPartnerState()) {
627 break;
628
629 case HA_READY_ST:
630 // If partner allocated new leases for which it didn't send lease updates
631 // to us we should synchronize our database.
632 if (communication_state_->hasPartnerNewUnsentUpdates()) {
634 } else {
635 // We did not miss any lease updates. There is no need to synchronize
636 // the database.
638 }
639 break;
640
641 case HA_TERMINATED_ST:
643 break;
644
645 case HA_DOWN_ST:
646 // @note: When partner is in HA_DOWN_ST, the server remains in
647 // HA_PARTNER_DOWN_ST state.
649 break;
650
651 default:
653 }
654}
655
656void
657HAService::partnerInMaintenanceStateHandler() {
658 // If we are transitioning from another state, we have to define new
659 // serving scopes appropriate for the new state. We don't do it if
660 // we remain in this state.
661 if (doOnEntry()) {
662 query_filter_.serveFailoverScopes();
663
665
666 // Log if the state machine is paused.
668
670 .arg(config_->getThisServerName());
671 }
672
674
675 if (isModelPaused()) {
677 return;
678 }
679
680 // Check if the clock skew is still acceptable. If not, transition to
681 // the terminated state.
682 if (shouldTerminate()) {
684 return;
685 }
686
687 if (isDHCPServiceDisabled()) {
689 return;
690 }
691
692 switch (communication_state_->getPartnerState()) {
693 case HA_DOWN_ST:
694 // @note: When partner is in HA_DOWN_ST, the server remains in
695 // HA_PARTNER_IN_MAINTENANCE_ST state.
697 break;
698
701 break;
702
703 default:
705 }
706}
707
708void
709HAService::passiveBackupStateHandler() {
710 // If we are transitioning from another state, we have to define new
711 // serving scopes appropriate for the new state. We don't do it if
712 // we remain in this state.
713 if (doOnEntry()) {
714 query_filter_.serveDefaultScopes();
716
717 // In the passive-backup state we don't send heartbeat.
718 communication_state_->stopHeartbeat();
719
720 // Log if the state machine is paused.
722 }
723
724 // There is nothing to do in that state. This server simply sends
725 // lease updates to the partners.
727}
728
729void
730HAService::readyStateHandler() {
731 // If we are transitioning from another state, we have to define new
732 // serving scopes appropriate for the new state. We don't do it if
733 // we remain in this state.
734 if (doOnEntry()) {
735 query_filter_.serveNoScopes();
737 communication_state_->clearRejectedLeaseUpdates();
738
739 // Log if the state machine is paused.
741 }
742
744
747 return;
748 }
749
750 // Check if the clock skew is still acceptable. If not, transition to
751 // the terminated state.
752 if (shouldTerminate()) {
754 return;
755 }
756
757 // Check if the partner state is valid per current configuration. If it is
758 // in an invalid state let's transition to the waiting state and stay there
759 // until the configuration is corrected.
760 if (isPartnerStateInvalid()) {
762 return;
763 }
764
765 switch (communication_state_->getPartnerState()) {
770 break;
771
774 break;
775
778 break;
779
780 case HA_READY_ST:
781 // If both servers are ready, the primary server "wins" and is
782 // transitioned first.
783 if (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::PRIMARY) {
786 } else {
788 }
789 break;
790
791 case HA_TERMINATED_ST:
793 break;
794
796 if (shouldPartnerDown()) {
798
799 } else {
801 }
802 break;
803
804 case HA_DOWN_ST:
806 break;
807
808 default:
810 }
811}
812
813void
814HAService::syncingStateHandler() {
815 // If we are transitioning from another state, we have to define new
816 // serving scopes appropriate for the new state. We don't do it if
817 // we remain in this state.
818 if (doOnEntry()) {
819 query_filter_.serveNoScopes();
821 communication_state_->clearRejectedLeaseUpdates();
822
823 // Log if the state machine is paused.
825 }
826
829 return;
830 }
831
832 // Check if the clock skew is still acceptable. If not, transition to
833 // the terminated state.
834 if (shouldTerminate()) {
836 return;
837 }
838
839 if (isDHCPServiceDisabled()) {
841 return;
842 }
843
844 // Check if the partner state is valid per current configuration. If it is
845 // in an invalid state let's transition to the waiting state and stay there
846 // until the configuration is corrected.
847 if (isPartnerStateInvalid()) {
849 return;
850 }
851
852 // We don't want to perform synchronous attempt to synchronize with
853 // a partner until we know that the partner is responding. Therefore,
854 // we wait for the heartbeat to complete successfully before we
855 // initiate the synchronization.
856 switch (communication_state_->getPartnerState()) {
857 case HA_TERMINATED_ST:
859 return;
860
862 // If the partner appears to be offline, let's transition to the partner
863 // down state. Otherwise, we'd be stuck trying to synchronize with a
864 // dead partner.
865 if (shouldPartnerDown()) {
867
868 } else {
870 }
871 break;
872
873 default:
874 // We don't want the heartbeat to interfere with the synchronization,
875 // so let's temporarily stop it.
876 communication_state_->stopHeartbeat();
877
878 // Timeout is configured in milliseconds. Need to convert to seconds.
879 unsigned int dhcp_disable_timeout =
880 static_cast<unsigned int>(config_->getSyncTimeout() / 1000);
881 if (dhcp_disable_timeout == 0) {
882 ++dhcp_disable_timeout;
883 }
884
885 // Perform synchronous leases update.
886 std::string status_message;
887 int sync_status = synchronize(status_message,
888 config_->getFailoverPeerConfig(),
889 dhcp_disable_timeout);
890
891 // If the leases synchronization was successful, let's transition
892 // to the ready state.
893 if (sync_status == CONTROL_RESULT_SUCCESS) {
895
896 } else {
897 // If the synchronization was unsuccessful we're back to the
898 // situation that the partner is unavailable and therefore
899 // we stay in the syncing state.
901 }
902 }
903
904 // Make sure that the heartbeat is re-enabled.
906}
907
908void
909HAService::terminatedStateHandler() {
910 // If we are transitioning from another state, we have to define new
911 // serving scopes appropriate for the new state. We don't do it if
912 // we remain in this state.
913 if (doOnEntry()) {
914 query_filter_.serveDefaultScopes();
916 communication_state_->clearRejectedLeaseUpdates();
917
918 // In the terminated state we don't send heartbeat.
919 communication_state_->stopHeartbeat();
920
921 // Log if the state machine is paused.
923
925 .arg(config_->getThisServerName());
926 }
927
929}
930
931void
932HAService::waitingStateHandler() {
933 // If we are transitioning from another state, we have to define new
934 // serving scopes appropriate for the new state. We don't do it if
935 // we remain in this state.
936 if (doOnEntry()) {
937 query_filter_.serveNoScopes();
939 communication_state_->clearRejectedLeaseUpdates();
940
941 // Log if the state machine is paused.
943 }
944
945 // Only schedule the heartbeat for non-backup servers.
946 if ((config_->getHAMode() != HAConfig::PASSIVE_BACKUP) &&
947 (config_->getThisServerConfig()->getRole() != HAConfig::PeerConfig::BACKUP)) {
949 }
950
953 return;
954 }
955
956 // Backup server must remain in its own state.
957 if (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::BACKUP) {
959 return;
960 }
961
962 // We're not a backup server, so we're either primary or secondary. If this is
963 // a passive-backup mode of operation, we're primary and we should transition
964 // to the passive-backup state.
965 if (config_->getHAMode() == HAConfig::PASSIVE_BACKUP) {
967 return;
968 }
969
970 // Check if the clock skew is still acceptable. If not, transition to
971 // the terminated state.
972 if (shouldTerminate()) {
974 return;
975 }
976
977 // Check if the partner state is valid per current configuration. If it is
978 // in an invalid state let's sit in the waiting state until the configuration
979 // is corrected.
980 if (isPartnerStateInvalid()) {
982 return;
983 }
984
985 switch (communication_state_->getPartnerState()) {
992 case HA_READY_ST:
993 // If we're configured to not synchronize lease database, proceed directly
994 // to the "ready" state.
995 verboseTransition(config_->amSyncingLeases() ? HA_SYNCING_ST : HA_READY_ST);
996 break;
997
998 case HA_SYNCING_ST:
1000 break;
1001
1002 case HA_TERMINATED_ST: {
1003 auto partner_in_terminated = communication_state_->getDurationSincePartnerStateTime();
1004 if (!partner_in_terminated.is_not_a_date_time() &&
1005 (partner_in_terminated.total_seconds()) / 60 >= HA_WAITING_TO_TERMINATED_ST_DELAY_MINUTES) {
1007 .arg(config_->getThisServerName())
1010 break;
1011 }
1012
1013 // We have checked above whether the clock skew is exceeding the threshold
1014 // and we should terminate. If we're here, it means that the clock skew
1015 // is acceptable. The partner may be still in the terminated state because
1016 // it hasn't been restarted yet. Probably, this server is the first one
1017 // being restarted after syncing the clocks. Let's just sit in the waiting
1018 // state until the partner gets restarted.
1020 .arg(config_->getThisServerName());
1022 break;
1023 }
1024 case HA_WAITING_ST:
1025 // If both servers are waiting, the primary server 'wins' and is
1026 // transitioned to the next state first.
1027 if (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::PRIMARY) {
1028 // If we're configured to not synchronize lease database, proceed directly
1029 // to the "ready" state.
1030 verboseTransition(config_->amSyncingLeases() ? HA_SYNCING_ST : HA_READY_ST);
1031
1032 } else {
1034 }
1035 break;
1036
1037 case HA_UNAVAILABLE_ST:
1038 if (shouldPartnerDown()) {
1040
1041 } else {
1043 }
1044 break;
1045
1046 default:
1048 }
1049}
1050
1051void
1052HAService::verboseTransition(const unsigned state) {
1053 // Get current and new state name.
1054 std::string current_state_name = getStateLabel(getCurrState());
1055 std::string new_state_name = getStateLabel(state);
1056
1057 // Turn them to upper case so as they are better visible in the logs.
1058 boost::to_upper(current_state_name);
1059 boost::to_upper(new_state_name);
1060
1061 if (config_->getHAMode() != HAConfig::PASSIVE_BACKUP) {
1062 // If this is load-balancing or hot-standby mode we also want to log
1063 // partner's state.
1064 auto partner_state = communication_state_->getPartnerState();
1065 std::string partner_state_name = getStateLabel(partner_state);
1066 boost::to_upper(partner_state_name);
1067
1068 // Log the transition.
1070 .arg(config_->getThisServerName())
1071 .arg(current_state_name)
1072 .arg(new_state_name)
1073 .arg(partner_state_name);
1074
1075 } else {
1076 // In the passive-backup mode we don't know the partner's state.
1078 .arg(config_->getThisServerName())
1079 .arg(current_state_name)
1080 .arg(new_state_name);
1081 }
1082
1083 // If we're transitioning directly from the "waiting" to "ready"
1084 // state it indicates that the database synchronization is
1085 // administratively disabled. Let's remind the user about this
1086 // configuration setting.
1087 if ((state == HA_READY_ST) && (getCurrState() == HA_WAITING_ST)) {
1089 .arg(config_->getThisServerName());
1090 }
1091
1092 // Do the actual transition.
1093 transition(state, getNextEvent());
1094
1095 // Inform the administrator whether or not lease updates are generated.
1096 // Updates are never generated by a backup server so it doesn't make
1097 // sense to log anything for the backup server.
1098 if ((config_->getHAMode() != HAConfig::PASSIVE_BACKUP) &&
1099 (config_->getThisServerConfig()->getRole() != HAConfig::PeerConfig::BACKUP)) {
1100 if (shouldSendLeaseUpdates(config_->getFailoverPeerConfig())) {
1102 .arg(config_->getThisServerName())
1103 .arg(new_state_name);
1104
1105 } else if (!config_->amSendingLeaseUpdates()) {
1106 // Lease updates are administratively disabled.
1108 .arg(config_->getThisServerName())
1109 .arg(new_state_name);
1110
1111 } else {
1112 // Lease updates are not administratively disabled, but they
1113 // are not issued because this is the backup server or because
1114 // in this state the server should not generate lease updates.
1116 .arg(config_->getThisServerName())
1117 .arg(new_state_name);
1118 }
1119 }
1120}
1121
1122int
1124 if (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::BACKUP) {
1125 return (HA_BACKUP_ST);
1126 }
1127
1128 switch (config_->getHAMode()) {
1130 return (HA_LOAD_BALANCING_ST);
1132 return (HA_HOT_STANDBY_ST);
1133 default:
1134 return (HA_PASSIVE_BACKUP_ST);
1135 }
1136}
1137
1138bool
1140 if (isModelPaused()) {
1142 .arg(config_->getThisServerName());
1143 unpauseModel();
1144 return (true);
1145 }
1146 return (false);
1147}
1148
1149void
1151 // Inform the administrator if the state machine is paused.
1152 if (isModelPaused()) {
1153 std::string state_name = stateToString(getCurrState());
1154 boost::to_upper(state_name);
1156 .arg(config_->getThisServerName())
1157 .arg(state_name);
1158 }
1159}
1160
1161void
1163 query_filter_.serveDefaultScopes();
1164}
1165
1166void
1168 query_filter_.serveFailoverScopes();
1169}
1170
1171bool
1173 return (inScopeInternal(query4));
1174}
1175
1176bool
1178 return (inScopeInternal(query6));
1179}
1180
1181template<typename QueryPtrType>
1182bool
1183HAService::inScopeInternal(QueryPtrType& query) {
1184 // Check if the query is in scope (should be processed by this server).
1185 std::string scope_class;
1186 const bool in_scope = query_filter_.inScope(query, scope_class);
1187 // Whether or not the query is going to be processed by this server,
1188 // we associate the query with the appropriate class.
1189 query->addClass(dhcp::ClientClass(scope_class));
1190 // The following is the part of the server failure detection algorithm.
1191 // If the query should be processed by the partner we need to check if
1192 // the partner responds. If the number of unanswered queries exceeds a
1193 // configured threshold, we will consider the partner to be offline.
1194 if (!in_scope && communication_state_->isCommunicationInterrupted()) {
1195 communication_state_->analyzeMessage(query);
1196 }
1197 // Indicate if the query is in scope.
1198 return (in_scope);
1199}
1200
1201bool
1203 return (shouldReclaimInternal(lease4));
1204}
1205
1206bool
1208 return (shouldReclaimInternal(lease6));
1209}
1210
1211template<typename LeasePtrType>
1212bool
1213HAService::shouldReclaimInternal(const LeasePtrType& lease) const {
1214 return (getCurrState() != HA_TERMINATED_ST || query_filter_.inScope(lease));
1215}
1216
1217void
1219 std::string current_state_name = getStateLabel(getCurrState());
1220 boost::to_upper(current_state_name);
1221
1222 // DHCP service should be enabled in the following states.
1223 const bool should_enable = ((getCurrState() == HA_COMMUNICATION_RECOVERY_ST) ||
1230
1231 if (!should_enable && network_state_->isServiceEnabled()) {
1232 current_state_name = getStateLabel(getCurrState());
1233 boost::to_upper(current_state_name);
1235 .arg(config_->getThisServerName())
1236 .arg(current_state_name);
1237 network_state_->disableService(getLocalOrigin());
1238
1239 } else if (should_enable && !network_state_->isServiceEnabled()) {
1240 current_state_name = getStateLabel(getCurrState());
1241 boost::to_upper(current_state_name);
1243 .arg(config_->getThisServerName())
1244 .arg(current_state_name);
1245 network_state_->enableService(getLocalOrigin());
1246 }
1247}
1248
1249bool
1251 // Checking whether the communication with the partner is OK is the
1252 // first step towards verifying if the server is up.
1253 if (communication_state_->isCommunicationInterrupted()) {
1254 // If the communication is interrupted, we also have to check
1255 // whether the partner answers DHCP requests. The only cases
1256 // when we don't (can't) do it are: the hot standby configuration
1257 // in which this server is a primary and when the DHCP service is
1258 // disabled so we can't analyze incoming traffic. Note that the
1259 // primary server can't check delayed responses to the partner
1260 // because the partner doesn't respond to any queries in this
1261 // configuration.
1262 if (network_state_->isServiceEnabled() &&
1263 ((config_->getHAMode() == HAConfig::LOAD_BALANCING) ||
1264 (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::STANDBY))) {
1265 return (communication_state_->failureDetected());
1266 }
1267
1268 // Hot standby / primary case.
1269 return (true);
1270 }
1271
1272 // Shouldn't transition to the partner down state.
1273 return (false);
1274}
1275
1276bool
1278 // Check if skew is fatally large.
1279 bool should_terminate = communication_state_->clockSkewShouldTerminate();
1280
1281 // If not issue a warning if it's getting large.
1282 if (!should_terminate) {
1283 communication_state_->clockSkewShouldWarn();
1284 // Check if we should terminate because the number of rejected leases
1285 // has been exceeded.
1286 should_terminate = communication_state_->rejectedLeaseUpdatesShouldTerminate();
1287 }
1288
1289 return (should_terminate);
1290}
1291
1292bool
1294 if (network_state_->disabledByDbConnection()) {
1295 return (true);
1296 }
1297 return (false);
1298}
1299
1300bool
1304
1305bool
1307 switch (communication_state_->getPartnerState()) {
1309 if (config_->getHAMode() != HAConfig::LOAD_BALANCING) {
1311 .arg(config_->getThisServerName());
1312 return (true);
1313 }
1314 break;
1315
1316 case HA_HOT_STANDBY_ST:
1317 if (config_->getHAMode() != HAConfig::HOT_STANDBY) {
1319 .arg(config_->getThisServerName());
1320 return (true);
1321 }
1322 break;
1323
1325 if (config_->getHAMode() != HAConfig::LOAD_BALANCING) {
1327 .arg(config_->getThisServerName());
1328 return (true);
1329 }
1330 break;
1331
1332 default:
1333 ;
1334 }
1335 return (false);
1336}
1337
1338size_t
1340 const dhcp::Lease4CollectionPtr& leases,
1341 const dhcp::Lease4CollectionPtr& deleted_leases,
1342 const hooks::ParkingLotHandlePtr& parking_lot) {
1343
1344 // Get configurations of the peers. Exclude this instance.
1345 HAConfig::PeerConfigMap peers_configs = config_->getOtherServersConfig();
1346
1347 size_t sent_num = 0;
1348
1349 // Schedule sending lease updates to each peer.
1350 for (auto const& p : peers_configs) {
1351 HAConfig::PeerConfigPtr conf = p.second;
1352
1353 // Check if the lease updates should be queued. This is the case when the
1354 // server is in the communication-recovery state. Queued lease updates may
1355 // be sent when the communication is re-established.
1356 if (shouldQueueLeaseUpdates(conf)) {
1357 // Lease updates for deleted leases.
1358 for (auto const& l : *deleted_leases) {
1359 // If a released lease is preserved in the database send the lease
1360 // update to the partner. Otherwise, delete the lease.
1361 if (l->state_ == Lease4::STATE_RELEASED) {
1363 } else {
1365 }
1366 }
1367
1368 // Lease updates for new allocations and updated leases.
1369 for (auto const& l : *leases) {
1371 }
1372
1373 continue;
1374 }
1375
1376 // Check if the lease update should be sent to the server. If we're in
1377 // the partner-down state we don't send lease updates to the partner.
1378 if (!shouldSendLeaseUpdates(conf)) {
1379 // If we decide to not send the lease updates to an active partner, we
1380 // should make a record of it in the communication state. The partner
1381 // can check if there were any unsent lease updates when he determines
1382 // whether it should synchronize its database or not when it recovers
1383 // from the partner-down state.
1384 if (conf->getRole() != HAConfig::PeerConfig::BACKUP) {
1385 communication_state_->increaseUnsentUpdateCount();
1386 }
1387 continue;
1388 }
1389
1390 // Lease updates for deleted leases.
1391 for (auto const& l : *deleted_leases) {
1392 // If a released lease is preserved in the database send the lease
1393 // update to the partner. Otherwise, delete the lease.
1394 if (l->state_ == Lease4::STATE_RELEASED) {
1396 parking_lot);
1397 } else {
1399 parking_lot);
1400 }
1401 }
1402
1403 // Lease updates for new allocations and updated leases.
1404 for (auto const& l : *leases) {
1406 parking_lot);
1407 }
1408
1409 // If we're contacting a backup server from which we don't expect a
1410 // response prior to responding to the DHCP client we don't count
1411 // it.
1412 if ((config_->amWaitingBackupAck() || (conf->getRole() != HAConfig::PeerConfig::BACKUP))) {
1413 ++sent_num;
1414 }
1415 }
1416
1417 return (sent_num);
1418}
1419
1420size_t
1422 const dhcp::Lease4Ptr& lease,
1423 const hooks::ParkingLotHandlePtr& parking_lot) {
1425 leases->push_back(lease);
1426 Lease4CollectionPtr deleted_leases(new Lease4Collection());
1427
1428 return (asyncSendLeaseUpdates(query, leases, deleted_leases, parking_lot));
1429}
1430
1431size_t
1433 const dhcp::Lease6CollectionPtr& leases,
1434 const dhcp::Lease6CollectionPtr& deleted_leases,
1435 const hooks::ParkingLotHandlePtr& parking_lot) {
1436
1437 // Get configurations of the peers. Exclude this instance.
1438 HAConfig::PeerConfigMap peers_configs = config_->getOtherServersConfig();
1439
1440 size_t sent_num = 0;
1441
1442 // Schedule sending lease updates to each peer.
1443 for (auto const& p : peers_configs) {
1444 HAConfig::PeerConfigPtr conf = p.second;
1445
1446 // Check if the lease updates should be queued. This is the case when the
1447 // server is in the communication-recovery state. Queued lease updates may
1448 // be sent when the communication is re-established.
1449 if (shouldQueueLeaseUpdates(conf)) {
1450 for (auto const& l : *deleted_leases) {
1451 // If a released lease is preserved in the database send the lease
1452 // update to the partner. Otherwise, delete the lease.
1453 if (l->state_ == Lease4::STATE_RELEASED) {
1455 } else {
1457 }
1458 }
1459
1460 // Lease updates for new allocations and updated leases.
1461 for (auto const& l : *leases) {
1463 }
1464
1465 continue;
1466 }
1467
1468 // Check if the lease update should be sent to the server. If we're in
1469 // the partner-down state we don't send lease updates to the partner.
1470 if (!shouldSendLeaseUpdates(conf)) {
1471 // If we decide to not send the lease updates to an active partner, we
1472 // should make a record of it in the communication state. The partner
1473 // can check if there were any unsent lease updates when he determines
1474 // whether it should synchronize its database or not when it recovers
1475 // from the partner-down state.
1476 if (conf->getRole() != HAConfig::PeerConfig::BACKUP) {
1477 communication_state_->increaseUnsentUpdateCount();
1478 }
1479 continue;
1480 }
1481
1482 // If we're contacting a backup server from which we don't expect a
1483 // response prior to responding to the DHCP client we don't count
1484 // it.
1485 if (config_->amWaitingBackupAck() || (conf->getRole() != HAConfig::PeerConfig::BACKUP)) {
1486 ++sent_num;
1487 }
1488
1489 // Send new/updated leases and deleted leases in one command.
1490 asyncSendLeaseUpdate(query, conf, CommandCreator::createLease6BulkApply(leases, deleted_leases),
1491 parking_lot);
1492 }
1493
1494 return (sent_num);
1495}
1496
1497template<typename QueryPtrType>
1498bool
1500 const ParkingLotHandlePtr& parking_lot) {
1501 if (MultiThreadingMgr::instance().getMode()) {
1502 std::lock_guard<std::mutex> lock(mutex_);
1503 return (leaseUpdateCompleteInternal(query, parking_lot));
1504 } else {
1505 return (leaseUpdateCompleteInternal(query, parking_lot));
1506 }
1507}
1508
1509template<typename QueryPtrType>
1510bool
1511HAService::leaseUpdateCompleteInternal(QueryPtrType& query,
1512 const ParkingLotHandlePtr& parking_lot) {
1513 auto it = pending_requests_.find(query);
1514
1515 // If there are no more pending requests for this query, let's unpark
1516 // the DHCP packet.
1517 if (it == pending_requests_.end() || (--pending_requests_[query] <= 0)) {
1518 if (parking_lot) {
1519 parking_lot->unpark(query);
1520 }
1521
1522 // If we have unparked the packet we can clear pending requests for
1523 // this query.
1524 if (it != pending_requests_.end()) {
1525 pending_requests_.erase(it);
1526 }
1527 return (true);
1528 }
1529 return (false);
1530}
1531
1532template<typename QueryPtrType>
1533void
1535 if (MultiThreadingMgr::instance().getMode()) {
1536 std::lock_guard<std::mutex> lock(mutex_);
1537 updatePendingRequestInternal(query);
1538 } else {
1539 updatePendingRequestInternal(query);
1540 }
1541}
1542
1543template<typename QueryPtrType>
1544void
1545HAService::updatePendingRequestInternal(QueryPtrType& query) {
1546 if (pending_requests_.count(query) == 0) {
1547 pending_requests_[query] = 1;
1548 } else {
1549 ++pending_requests_[query];
1550 }
1551}
1552
1553template<typename QueryPtrType>
1554void
1555HAService::asyncSendLeaseUpdate(const QueryPtrType& query,
1557 const ConstElementPtr& command,
1558 const ParkingLotHandlePtr& parking_lot) {
1559 // Create HTTP/1.1 request including our command.
1560 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
1562 HostHttpHeader(config->getUrl().getStrippedHostname()));
1563 config->addBasicAuthHttpHeader(request);
1564 request->setBodyAsJson(command);
1565 request->finalize();
1566
1567 // Response object should also be created because the HTTP client needs
1568 // to know the type of the expected response.
1569 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
1570
1571 // When possible we prefer to pass weak pointers to the queries, rather
1572 // than shared pointers, to avoid memory leaks in case cross reference
1573 // between the pointers.
1574 boost::weak_ptr<typename QueryPtrType::element_type> weak_query(query);
1575
1576 // Schedule asynchronous HTTP request.
1577 client_->asyncSendRequest(config->getUrl(), config->getTlsContext(),
1578 request, response,
1579 [this, weak_query, parking_lot, config]
1580 (const boost::system::error_code& ec,
1582 const std::string& error_str) {
1583 // Get the shared pointer of the query. The server should keep the
1584 // pointer to the query and then park it. Therefore, we don't really
1585 // expect it to be null. If it is null, something is really wrong.
1586 QueryPtrType query_ptr = weak_query.lock();
1587 if (!query_ptr) {
1588 isc_throw(Unexpected, "query is null while receiving response from"
1589 " HA peer. This is programmatic error");
1590 }
1591
1592 // There are four possible groups of errors during the lease update.
1593 // One is the IO error causing issues in communication with the peer.
1594 // Another one is an HTTP parsing error. The third type occurs when
1595 // the partner receives the command but it is invalid or there is
1596 // an internal processing error. Finally, the forth type is when the
1597 // conflict status code is returned in the response indicating that
1598 // the lease update does not match the partner's configuration.
1599
1600 bool lease_update_success = true;
1601 bool lease_update_conflict = false;
1602
1603 // Handle first two groups of errors.
1604 if (ec || !error_str.empty()) {
1605 LOG_WARN(ha_logger, HA_LEASE_UPDATE_COMMUNICATIONS_FAILED)
1606 .arg(config_->getThisServerName())
1607 .arg(query_ptr->getLabel())
1608 .arg(config->getLogLabel())
1609 .arg(ec ? ec.message() : error_str);
1610
1611 // Communication error, so let's drop parked packet. The DHCP
1612 // response will not be sent.
1613 lease_update_success = false;
1614
1615 } else {
1616
1617 try {
1618 int rcode = 0;
1619 auto args = verifyAsyncResponse(http_response, rcode);
1620 // In the v6 case the server may return a list of failed lease
1621 // updates and we should log them.
1622 logFailedLeaseUpdates(query_ptr, args);
1623
1624 } catch (const ConflictError& ex) {
1625 // Handle forth group of errors.
1626 lease_update_conflict = true;
1627 lease_update_success = false;
1628 communication_state_->reportRejectedLeaseUpdate(query_ptr);
1629
1631 .arg(config_->getThisServerName())
1632 .arg(query_ptr->getLabel())
1633 .arg(config->getLogLabel())
1634 .arg(ex.what());
1635
1636 } catch (const std::exception& ex) {
1637 // Handle third group of errors.
1639 .arg(config_->getThisServerName())
1640 .arg(query_ptr->getLabel())
1641 .arg(config->getLogLabel())
1642 .arg(ex.what());
1643
1644 // Error while doing an update. The DHCP response will not be sent.
1645 lease_update_success = false;
1646 }
1647 }
1648
1649 // We don't care about the result of the lease update to the backup server.
1650 // It is a best effort update.
1651 if (config->getRole() != HAConfig::PeerConfig::BACKUP) {
1652 // If the lease update was unsuccessful we may need to set the partner
1653 // state as unavailable.
1654 if (!lease_update_success) {
1655 // Do not set it as unavailable if it was a conflict because the
1656 // partner actually responded.
1657 if (!lease_update_conflict) {
1658 // If we were unable to communicate with the partner we set partner's
1659 // state as unavailable.
1660 communication_state_->setPartnerUnavailable();
1661 }
1662 } else {
1663 // Lease update successful and we may need to clear some previously
1664 // rejected lease updates.
1665 communication_state_->reportSuccessfulLeaseUpdate(query_ptr);
1666 }
1667 }
1668
1669 // It is possible to configure the server to not wait for a response from
1670 // the backup server before we unpark the packet and respond to the client.
1671 // Here we check if we're dealing with such situation.
1672 if (config_->amWaitingBackupAck() || (config->getRole() != HAConfig::PeerConfig::BACKUP)) {
1673 // We're expecting a response from the backup server or it is not
1674 // a backup server and the lease update was unsuccessful. In such
1675 // case the DHCP exchange fails.
1676 if (!lease_update_success) {
1677 if (parking_lot) {
1678 parking_lot->drop(query_ptr);
1679 }
1680 }
1681 } else {
1682 // This was a response from the backup server and we're configured to
1683 // not wait for their acknowledgments, so there is nothing more to do.
1684 return;
1685 }
1686
1687 if (leaseUpdateComplete(query_ptr, parking_lot)) {
1688 // If we have finished sending the lease updates we need to run the
1689 // state machine until the state machine finds that additional events
1690 // are required, such as next heartbeat or a lease update. The runModel()
1691 // may transition to another state, schedule asynchronous tasks etc.
1692 // Then it returns control to the DHCP server.
1693 runModel(HA_LEASE_UPDATES_COMPLETE_EVT);
1694 }
1695 },
1697 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
1698 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
1699 std::bind(&HAService::clientCloseHandler, this, ph::_1)
1700 );
1701
1702 // The number of pending requests is the number of requests for which we
1703 // expect an acknowledgment prior to responding to the DHCP clients. If
1704 // we're configured to wait for the acks from the backups or it is not
1705 // a backup increase the number of pending requests.
1706 if (config_->amWaitingBackupAck() || (config->getRole() != HAConfig::PeerConfig::BACKUP)) {
1707 // Request scheduled, so update the request counters for the query.
1708 updatePendingRequest(query);
1709 }
1710}
1711
1712bool
1714 // Never send lease updates if they are administratively disabled.
1715 if (!config_->amSendingLeaseUpdates()) {
1716 return (false);
1717 }
1718
1719 // Always send updates to the backup server.
1720 if (peer_config->getRole() == HAConfig::PeerConfig::BACKUP) {
1721 return (true);
1722 }
1723
1724 // Never send updates if this is a backup server.
1725 if (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::BACKUP) {
1726 return (false);
1727 }
1728
1729 // In other case, whether we send lease updates or not depends on our
1730 // state.
1731 switch (getCurrState()) {
1732 case HA_HOT_STANDBY_ST:
1735 return (true);
1736
1737 case HA_DOWN_ST:
1738 return (false);
1739
1740 default:
1741 ;
1742 }
1743
1744 return (false);
1745}
1746
1747bool
1749 if (!config_->amSendingLeaseUpdates()) {
1750 return (false);
1751 }
1752
1753 if (peer_config->getRole() == HAConfig::PeerConfig::BACKUP) {
1754 return (false);
1755 }
1756
1758}
1759
1760void
1762 const ConstElementPtr& args) const {
1763 // If there are no arguments, it means that the update was successful.
1764 if (!args || (args->getType() != Element::map)) {
1765 return;
1766 }
1767
1768 // Instead of duplicating the code between the failed-deleted-leases and
1769 // failed-leases, let's just have one function that does it for both.
1770 auto log_proc = [query, args](const std::string& param_name, const log::MessageID& mesid) {
1771 // Check if there are any failed leases.
1772 auto failed_leases = args->get(param_name);
1773
1774 // The failed leases must be a list.
1775 if (failed_leases && (failed_leases->getType() == Element::list)) {
1776 // Go over the failed leases and log each of them.
1777 for (unsigned i = 0; i < failed_leases->size(); ++i) {
1778 auto lease = failed_leases->get(i);
1779 if (lease->getType() == Element::map) {
1780
1781 // ip-address
1782 auto ip_address = lease->get("ip-address");
1783
1784 // lease type
1785 auto lease_type = lease->get("type");
1786
1787 // error-message
1788 auto error_message = lease->get("error-message");
1789
1790 LOG_INFO(ha_logger, mesid)
1791 .arg(query->getLabel())
1792 .arg(lease_type && (lease_type->getType() == Element::string) ?
1793 lease_type->stringValue() : "(unknown)")
1794 .arg(ip_address && (ip_address->getType() == Element::string) ?
1795 ip_address->stringValue() : "(unknown)")
1796 .arg(error_message && (error_message->getType() == Element::string) ?
1797 error_message->stringValue() : "(unknown)");
1798 }
1799 }
1800 }
1801 };
1802
1803 // Process "failed-deleted-leases"
1804 log_proc("failed-deleted-leases", HA_LEASE_UPDATE_DELETE_FAILED_ON_PEER);
1805
1806 // Process "failed-leases".
1807 log_proc("failed-leases", HA_LEASE_UPDATE_CREATE_UPDATE_FAILED_ON_PEER);
1808}
1809
1812 ElementPtr ha_servers = Element::createMap();
1813
1814 // Local part
1817 role = config_->getThisServerConfig()->getRole();
1818 std::string role_txt = HAConfig::PeerConfig::roleToString(role);
1819 local->set("role", Element::create(role_txt));
1820 int state = getCurrState();
1821 try {
1822 local->set("state", Element::create(stateToString(state)));
1823
1824 } catch (...) {
1825 // Empty string on error.
1826 local->set("state", Element::create(std::string()));
1827 }
1828 std::set<std::string> scopes = query_filter_.getServedScopes();
1830 for (auto const& scope : scopes) {
1831 list->add(Element::create(scope));
1832 }
1833 local->set("scopes", list);
1834 local->set("server-name", Element::create(config_->getThisServerName()));
1835 auto const my_time(communication_state_->getMyTimeAtSkew());
1836 if (my_time.is_not_a_date_time()) {
1837 local->set("system-time", Element::create());
1838 } else {
1839 local->set("system-time", Element::create(ptimeToText(my_time, 0)));
1840 }
1841 ha_servers->set("local", local);
1842
1843 // Do not include remote server information if this is a backup server or
1844 // we're in the passive-backup mode.
1845 if ((config_->getHAMode() == HAConfig::PASSIVE_BACKUP) ||
1846 (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::BACKUP)) {
1847 return (ha_servers);
1848 }
1849
1850 // Remote part
1851 ElementPtr remote = communication_state_->getReport();
1852
1853 try {
1854 role = config_->getFailoverPeerConfig()->getRole();
1855 role_txt = HAConfig::PeerConfig::roleToString(role);
1856 remote->set("role", Element::create(role_txt));
1857
1858 } catch (...) {
1859 remote->set("role", Element::create(std::string()));
1860 }
1861 remote->set("server-name", Element::create(config_->getFailoverPeerConfig()->getName()));
1862 ha_servers->set("remote", remote);
1863
1864 return (ha_servers);
1865}
1866
1869 ElementPtr arguments = Element::createMap();
1870 std::string state_label = getState(getCurrState())->getLabel();
1871 arguments->set("state", Element::create(state_label));
1872
1873 std::string date_time = HttpDateTime().rfc1123Format();
1874 arguments->set("date-time", Element::create(date_time));
1875
1876 auto scopes = query_filter_.getServedScopes();
1877 ElementPtr scopes_list = Element::createList();
1878 for (auto const& scope : scopes) {
1879 scopes_list->add(Element::create(scope));
1880 }
1881 arguments->set("scopes", scopes_list);
1882
1883 arguments->set("unsent-update-count",
1884 Element::create(static_cast<int64_t>(communication_state_->getUnsentUpdateCount())));
1885
1886 return (createAnswer(CONTROL_RESULT_SUCCESS, "HA peer status returned.",
1887 arguments));
1888}
1889
1892 if (getCurrState() == HA_WAITING_ST) {
1893 return (createAnswer(CONTROL_RESULT_SUCCESS, "HA state machine already in WAITING state."));
1894 }
1897 return (createAnswer(CONTROL_RESULT_SUCCESS, "HA state machine reset."));
1898}
1899
1900void
1902 HAConfig::PeerConfigPtr partner_config = config_->getFailoverPeerConfig();
1903
1904 // If the sync_complete_notified_ is true it means that the partner
1905 // notified us that it had completed lease database synchronization.
1906 // We confirm that the partner is operational by sending the heartbeat
1907 // to it. Regardless if the partner responds to our heartbeats or not,
1908 // we should clear this flag. But, since we need the current value in
1909 // the async call handler, we save it in the local variable before
1910 // clearing it.
1911 bool sync_complete_notified = sync_complete_notified_;
1913
1914 // Create HTTP/1.1 request including our command.
1915 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
1917 HostHttpHeader(partner_config->getUrl().getStrippedHostname()));
1918 partner_config->addBasicAuthHttpHeader(request);
1919 request->setBodyAsJson(CommandCreator::createHeartbeat(config_->getThisServerName(),
1920 server_type_));
1921 request->finalize();
1922
1923 // Response object should also be created because the HTTP client needs
1924 // to know the type of the expected response.
1925 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
1926
1927 // Schedule asynchronous HTTP request.
1928 client_->asyncSendRequest(partner_config->getUrl(),
1929 partner_config->getTlsContext(),
1930 request, response,
1931 [this, partner_config, sync_complete_notified]
1932 (const boost::system::error_code& ec,
1934 const std::string& error_str) {
1935
1936 // There are three possible groups of errors during the heartbeat.
1937 // One is the IO error causing issues in communication with the peer.
1938 // Another one is an HTTP parsing error. The last type of error is
1939 // when non-success error code is returned in the response carried
1940 // in the HTTP message or if the JSON response is otherwise broken.
1941
1942 bool heartbeat_success = true;
1943
1944 // Handle first two groups of errors.
1945 if (ec || !error_str.empty()) {
1946 LOG_WARN(ha_logger, HA_HEARTBEAT_COMMUNICATIONS_FAILED)
1947 .arg(config_->getThisServerName())
1948 .arg(partner_config->getLogLabel())
1949 .arg(ec ? ec.message() : error_str);
1950 heartbeat_success = false;
1951
1952 } else {
1953
1954 // Handle third group of errors.
1955 try {
1956 // Response must contain arguments and the arguments must
1957 // be a map.
1958 int rcode = 0;
1959 ConstElementPtr args = verifyAsyncResponse(http_response, rcode);
1960 if (!args || args->getType() != Element::map) {
1961 isc_throw(CtrlChannelError, "returned arguments in the response"
1962 " must be a map");
1963 }
1964 // Response must include partner's state.
1965 ConstElementPtr state = args->get("state");
1966 if (!state || state->getType() != Element::string) {
1967 isc_throw(CtrlChannelError, "server state not returned in response"
1968 " to a ha-heartbeat command or it is not a string");
1969 }
1970 // Remember the partner's state. This may throw if the returned
1971 // state is invalid.
1972 communication_state_->setPartnerState(state->stringValue());
1973
1974 ConstElementPtr date_time = args->get("date-time");
1975 if (!date_time || date_time->getType() != Element::string) {
1976 isc_throw(CtrlChannelError, "date-time not returned in response"
1977 " to a ha-heartbeat command or it is not a string");
1978 }
1979 // Note the time returned by the partner to calculate the clock skew.
1980 communication_state_->setPartnerTime(date_time->stringValue());
1981
1982 // Remember the scopes served by the partner.
1983 try {
1984 auto scopes = args->get("scopes");
1985 communication_state_->setPartnerScopes(scopes);
1986
1987 } catch (...) {
1988 // We don't want to fail if the scopes are missing because
1989 // this would be incompatible with old HA hook library
1990 // versions. We may make it mandatory one day, but during
1991 // upgrades of existing HA setup it would be a real issue
1992 // if we failed here.
1993 }
1994
1995 // unsent-update-count was not present in earlier HA versions.
1996 // Let's check if the partner has sent the parameter. We initialized
1997 // the counter to 0, and it remains 0 if the partner doesn't send it.
1998 // It effectively means that we don't track partner's unsent updates
1999 // as in the earlier HA versions.
2000 auto unsent_update_count = args->get("unsent-update-count");
2001 if (unsent_update_count) {
2002 if (unsent_update_count->getType() != Element::integer) {
2003 isc_throw(CtrlChannelError, "unsent-update-count returned in"
2004 " the ha-heartbeat response is not an integer");
2005 }
2006 communication_state_->setPartnerUnsentUpdateCount(static_cast<uint64_t>
2007 (unsent_update_count->intValue()));
2008 }
2009
2010 } catch (const std::exception& ex) {
2012 .arg(config_->getThisServerName())
2013 .arg(partner_config->getLogLabel())
2014 .arg(ex.what());
2015 heartbeat_success = false;
2016 }
2017 }
2018
2019 // If heartbeat was successful, let's mark the connection with the
2020 // peer as healthy.
2021 if (heartbeat_success) {
2022 communication_state_->poke();
2023
2024 } else {
2025 // We were unable to retrieve partner's state, so let's mark it
2026 // as unavailable.
2027 communication_state_->setPartnerUnavailable();
2028 // Log if the communication is interrupted.
2029 if (communication_state_->isCommunicationInterrupted()) {
2030 LOG_WARN(ha_logger, HA_COMMUNICATION_INTERRUPTED)
2031 .arg(config_->getThisServerName())
2032 .arg(partner_config->getName());
2033 }
2034 }
2035
2036 startHeartbeat();
2037 // Even though the partner notified us about the synchronization completion,
2038 // we still can't communicate with the partner. Let's continue serving
2039 // the clients until the link is fixed.
2040 if (sync_complete_notified && !heartbeat_success) {
2041 postNextEvent(HA_SYNCED_PARTNER_UNAVAILABLE_EVT);
2042 }
2043 // Whatever the result of the heartbeat was, the state machine needs
2044 // to react to this. Let's run the state machine until the state machine
2045 // finds that some new events are required, i.e. next heartbeat or
2046 // lease update. The runModel() may transition to another state, schedule
2047 // asynchronous tasks etc. Then it returns control to the DHCP server.
2048 runModel(HA_HEARTBEAT_COMPLETE_EVT);
2049 },
2051 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
2052 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
2053 std::bind(&HAService::clientCloseHandler, this, ph::_1)
2054 );
2055}
2056
2057void
2059 if (!communication_state_->isHeartbeatRunning()) {
2061 }
2062}
2063
2064void
2066 if (config_->getHeartbeatDelay() > 0) {
2067 communication_state_->startHeartbeat(config_->getHeartbeatDelay(),
2069 this));
2070 }
2071}
2072
2073void
2075 const HAConfig::PeerConfigPtr& remote_config,
2076 const unsigned int max_period,
2077 PostRequestCallback post_request_action) {
2078 // Create HTTP/1.1 request including our command.
2079 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2081 HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
2082
2083 remote_config->addBasicAuthHttpHeader(request);
2084 request->setBodyAsJson(CommandCreator::createDHCPDisable(getRemoteOrigin(),
2085 max_period,
2086 server_type_));
2087 request->finalize();
2088
2089 // Response object should also be created because the HTTP client needs
2090 // to know the type of the expected response.
2091 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2092
2093 // Schedule asynchronous HTTP request.
2094 http_client.asyncSendRequest(remote_config->getUrl(),
2095 remote_config->getTlsContext(),
2096 request, response,
2097 [this, remote_config, post_request_action]
2098 (const boost::system::error_code& ec,
2100 const std::string& error_str) {
2101
2102 // There are three possible groups of errors during the heartbeat.
2103 // One is the IO error causing issues in communication with the peer.
2104 // Another one is an HTTP parsing error. The last type of error is
2105 // when non-success error code is returned in the response carried
2106 // in the HTTP message or if the JSON response is otherwise broken.
2107
2108 int rcode = 0;
2109 std::string error_message;
2110
2111 // Handle first two groups of errors.
2112 if (ec || !error_str.empty()) {
2113 error_message = (ec ? ec.message() : error_str);
2114 LOG_ERROR(ha_logger, HA_DHCP_DISABLE_COMMUNICATIONS_FAILED)
2115 .arg(config_->getThisServerName())
2116 .arg(remote_config->getLogLabel())
2117 .arg(error_message);
2118
2119 } else {
2120
2121 // Handle third group of errors.
2122 try {
2123 static_cast<void>(verifyAsyncResponse(http_response, rcode));
2124
2125 } catch (const std::exception& ex) {
2126 error_message = ex.what();
2128 .arg(config_->getThisServerName())
2129 .arg(remote_config->getLogLabel())
2130 .arg(error_message);
2131 }
2132 }
2133
2134 // If there was an error communicating with the partner, mark the
2135 // partner as unavailable.
2136 if (!error_message.empty()) {
2137 communication_state_->setPartnerUnavailable();
2138 }
2139
2140 // Invoke post request action if it was specified.
2141 if (post_request_action) {
2142 post_request_action(error_message.empty(),
2143 error_message,
2144 rcode);
2145 }
2146 },
2148 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
2149 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
2150 std::bind(&HAService::clientCloseHandler, this, ph::_1)
2151 );
2152}
2153
2154void
2156 const HAConfig::PeerConfigPtr& remote_config,
2157 PostRequestCallback post_request_action) {
2158 // Create HTTP/1.1 request including our command.
2159 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2161 HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
2162 remote_config->addBasicAuthHttpHeader(request);
2163 request->setBodyAsJson(CommandCreator::createDHCPEnable(getRemoteOrigin(),
2164 server_type_));
2165 request->finalize();
2166
2167 // Response object should also be created because the HTTP client needs
2168 // to know the type of the expected response.
2169 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2170
2171 // Schedule asynchronous HTTP request.
2172 http_client.asyncSendRequest(remote_config->getUrl(),
2173 remote_config->getTlsContext(),
2174 request, response,
2175 [this, remote_config, post_request_action]
2176 (const boost::system::error_code& ec,
2178 const std::string& error_str) {
2179
2180 // There are three possible groups of errors during the heartbeat.
2181 // One is the IO error causing issues in communication with the peer.
2182 // Another one is an HTTP parsing error. The last type of error is
2183 // when non-success error code is returned in the response carried
2184 // in the HTTP message or if the JSON response is otherwise broken.
2185
2186 int rcode = 0;
2187 std::string error_message;
2188
2189 // Handle first two groups of errors.
2190 if (ec || !error_str.empty()) {
2191 error_message = (ec ? ec.message() : error_str);
2192 LOG_ERROR(ha_logger, HA_DHCP_ENABLE_COMMUNICATIONS_FAILED)
2193 .arg(config_->getThisServerName())
2194 .arg(remote_config->getLogLabel())
2195 .arg(error_message);
2196
2197 } else {
2198
2199 // Handle third group of errors.
2200 try {
2201 static_cast<void>(verifyAsyncResponse(http_response, rcode));
2202
2203 } catch (const std::exception& ex) {
2204 error_message = ex.what();
2206 .arg(config_->getThisServerName())
2207 .arg(remote_config->getLogLabel())
2208 .arg(error_message);
2209 }
2210 }
2211
2212 // If there was an error communicating with the partner, mark the
2213 // partner as unavailable.
2214 if (!error_message.empty()) {
2215 communication_state_->setPartnerUnavailable();
2216 }
2217
2218 // Invoke post request action if it was specified.
2219 if (post_request_action) {
2220 post_request_action(error_message.empty(),
2221 error_message,
2222 rcode);
2223 }
2224 },
2226 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
2227 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
2228 std::bind(&HAService::clientCloseHandler, this, ph::_1)
2229 );
2230}
2231
2232void
2234 network_state_->disableService(getLocalOrigin());
2235}
2236
2237void
2239 network_state_->enableService(getLocalOrigin());
2240}
2241
2242void
2244 PostSyncCallback null_action;
2245
2246 // Timeout is configured in milliseconds. Need to convert to seconds.
2247 unsigned int dhcp_disable_timeout =
2248 static_cast<unsigned int>(config_->getSyncTimeout() / 1000);
2249 if (dhcp_disable_timeout == 0) {
2250 // Ensure that we always use at least 1 second timeout.
2251 dhcp_disable_timeout = 1;
2252 }
2253
2254 lease_sync_filter_.apply();
2255 asyncSyncLeases(*client_, config_->getFailoverPeerConfig(),
2256 dhcp_disable_timeout, LeasePtr(), null_action);
2257}
2258
2259void
2261 const HAConfig::PeerConfigPtr& remote_config,
2262 const unsigned int max_period,
2263 const dhcp::LeasePtr& last_lease,
2264 PostSyncCallback post_sync_action,
2265 const bool dhcp_disabled) {
2266 // Synchronization starts with a command to disable DHCP service of the
2267 // peer from which we're fetching leases. We don't want the other server
2268 // to allocate new leases while we fetch from it. The DHCP service will
2269 // be disabled for a certain amount of time and will be automatically
2270 // re-enabled if we die during the synchronization.
2271 asyncDisableDHCPService(http_client, remote_config, max_period,
2272 [this, &http_client, remote_config, max_period, last_lease,
2273 post_sync_action, dhcp_disabled]
2274 (const bool success, const std::string& error_message, const int) {
2275
2276 // If we have successfully disabled the DHCP service on the peer,
2277 // we can start fetching the leases.
2278 if (success) {
2279 // The last argument indicates that disabling the DHCP
2280 // service on the partner server was successful.
2281 asyncSyncLeasesInternal(http_client, remote_config, max_period,
2282 last_lease, post_sync_action, true);
2283
2284 } else {
2285 post_sync_action(success, error_message, dhcp_disabled);
2286 }
2287 });
2288}
2289
2290void
2292 const HAConfig::PeerConfigPtr& remote_config,
2293 const unsigned int max_period,
2294 const dhcp::LeasePtr& last_lease,
2295 PostSyncCallback post_sync_action,
2296 const bool dhcp_disabled) {
2297 // Create HTTP/1.1 request including our command.
2298 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2300 HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
2301 remote_config->addBasicAuthHttpHeader(request);
2303 request->setBodyAsJson(CommandCreator::createLease4GetPage(
2304 boost::dynamic_pointer_cast<Lease4>(last_lease), config_->getSyncPageLimit()));
2305
2306 } else {
2307 request->setBodyAsJson(CommandCreator::createLease6GetPage(
2308 boost::dynamic_pointer_cast<Lease6>(last_lease), config_->getSyncPageLimit()));
2309 }
2310 request->finalize();
2311
2312 // Response object should also be created because the HTTP client needs
2313 // to know the type of the expected response.
2314 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2315
2316 // Schedule asynchronous HTTP request.
2317 http_client.asyncSendRequest(remote_config->getUrl(),
2318 remote_config->getTlsContext(),
2319 request, response,
2320 [this, remote_config, post_sync_action, &http_client, max_period, dhcp_disabled]
2321 (const boost::system::error_code& ec,
2323 const std::string& error_str) {
2324
2325 // Holds last lease received on the page of leases. If the last
2326 // page was hit, this value remains null.
2327 LeasePtr last_lease_in_callback;
2328
2329 // There are three possible groups of errors during the heartbeat.
2330 // One is the IO error causing issues in communication with the peer.
2331 // Another one is an HTTP parsing error. The last type of error is
2332 // when non-success error code is returned in the response carried
2333 // in the HTTP message or if the JSON response is otherwise broken.
2334
2335 std::string error_message;
2336
2337 // Handle first two groups of errors.
2338 if (ec || !error_str.empty()) {
2339 error_message = (ec ? ec.message() : error_str);
2340 LOG_ERROR(ha_logger, HA_LEASES_SYNC_COMMUNICATIONS_FAILED)
2341 .arg(config_->getThisServerName())
2342 .arg(remote_config->getLogLabel())
2343 .arg(error_message);
2344
2345 } else {
2346 // Handle third group of errors.
2347 try {
2348 int rcode = 0;
2349 ConstElementPtr args = verifyAsyncResponse(http_response, rcode);
2350
2351 // Arguments must be a map.
2352 if (args && (args->getType() != Element::map)) {
2353 isc_throw(CtrlChannelError,
2354 "arguments in the received response must be a map");
2355 }
2356
2357 ConstElementPtr leases = args->get("leases");
2358 if (!leases || (leases->getType() != Element::list)) {
2359 isc_throw(CtrlChannelError,
2360 "server response does not contain leases argument or this"
2361 " argument is not a list");
2362 }
2363
2364 // Iterate over the leases and update the database as appropriate.
2365 auto const& leases_element = leases->listValue();
2366
2367 LOG_INFO(ha_logger, HA_LEASES_SYNC_LEASE_PAGE_RECEIVED)
2368 .arg(config_->getThisServerName())
2369 .arg(leases_element.size())
2370 .arg(remote_config->getLogLabel());
2371
2372 // Count actually applied leases.
2373 uint64_t applied_lease_count = 0;
2374 for (auto l = leases_element.begin(); l != leases_element.end(); ++l) {
2375 try {
2376
2377 if (server_type_ == HAServerType::DHCPv4) {
2378 Lease4Ptr lease = Lease4::fromElement(*l);
2379
2380 // If we're not on the last page and we're processing final lease on
2381 // this page, let's record the lease as input to the next
2382 // lease4-get-page command.
2383 if ((leases_element.size() >= config_->getSyncPageLimit()) &&
2384 (l + 1 == leases_element.end())) {
2385 last_lease_in_callback = boost::dynamic_pointer_cast<Lease>(lease);
2386 }
2387
2388 if (!lease_sync_filter_.shouldSync(lease)) {
2389 continue;
2390 }
2391
2392 // Check if there is such lease in the database already.
2393 Lease4Ptr existing_lease = LeaseMgrFactory::instance().getLease4(lease->addr_);
2394 if (!existing_lease) {
2395 // There is no such lease, so let's add it.
2396 LeaseMgrFactory::instance().addLease(lease);
2397 ++applied_lease_count;
2398 LeaseMgr::updateStatsOnAdd(lease);
2399 } else if (existing_lease->cltt_ < lease->cltt_) {
2400 // If the existing lease is older than the fetched lease, update
2401 // the lease in our local database.
2402 // Update lease current expiration time with value received from the
2403 // database. Some database backends reject operations on the lease if
2404 // the current expiration time value does not match what is stored.
2405 Lease::syncCurrentExpirationTime(*existing_lease, *lease);
2406 LeaseMgrFactory::instance().updateLease4(lease);
2407 ++applied_lease_count;
2408 LeaseMgr::updateStatsOnUpdate(existing_lease, lease);
2409 } else {
2410 LOG_DEBUG(ha_logger, DBGLVL_TRACE_BASIC, HA_LEASE_SYNC_STALE_LEASE4_SKIP)
2411 .arg(config_->getThisServerName())
2412 .arg(lease->addr_.toText())
2413 .arg(lease->subnet_id_);
2414 }
2415
2416 } else {
2417 Lease6Ptr lease = Lease6::fromElement(*l);
2418
2419 // If we're not on the last page and we're processing final lease on
2420 // this page, let's record the lease as input to the next
2421 // lease6-get-page command.
2422 if ((leases_element.size() >= config_->getSyncPageLimit()) &&
2423 (l + 1 == leases_element.end())) {
2424 last_lease_in_callback = boost::dynamic_pointer_cast<Lease>(lease);
2425 }
2426
2427 if (!lease_sync_filter_.shouldSync(lease)) {
2428 continue;
2429 }
2430
2431 // Check if there is such lease in the database already.
2432 Lease6Ptr existing_lease = LeaseMgrFactory::instance().getLease6(lease->type_,
2433 lease->addr_);
2434 if (!existing_lease) {
2435 // There is no such lease, so let's add it.
2436 LeaseMgrFactory::instance().addLease(lease);
2437 ++applied_lease_count;
2438 LeaseMgr::updateStatsOnAdd(lease);
2439 } else if (existing_lease->cltt_ < lease->cltt_) {
2440 // If the existing lease is older than the fetched lease, update
2441 // the lease in our local database.
2442 // Update lease current expiration time with value received from the
2443 // database. Some database backends reject operations on the lease if
2444 // the current expiration time value does not match what is stored.
2445 Lease::syncCurrentExpirationTime(*existing_lease, *lease);
2446 LeaseMgrFactory::instance().updateLease6(lease);
2447 ++applied_lease_count;
2448 LeaseMgr::updateStatsOnUpdate(existing_lease, lease);
2449 } else {
2450 LOG_DEBUG(ha_logger, DBGLVL_TRACE_BASIC, HA_LEASE_SYNC_STALE_LEASE6_SKIP)
2451 .arg(config_->getThisServerName())
2452 .arg(lease->addr_.toText())
2453 .arg(lease->subnet_id_);
2454 }
2455 }
2456
2457 } catch (const std::exception& ex) {
2458 LOG_WARN(ha_logger, HA_LEASE_SYNC_FAILED)
2459 .arg(config_->getThisServerName())
2460 .arg((*l)->str())
2461 .arg(ex.what());
2462 }
2463 }
2464
2465 LOG_INFO(ha_logger, HA_LEASES_SYNC_APPLIED_LEASES)
2466 .arg(config_->getThisServerName())
2467 .arg(applied_lease_count);
2468
2469 } catch (const std::exception& ex) {
2470 error_message = ex.what();
2472 .arg(config_->getThisServerName())
2473 .arg(remote_config->getLogLabel())
2474 .arg(error_message);
2475 }
2476 }
2477
2478 // If there was an error communicating with the partner, mark the
2479 // partner as unavailable.
2480 if (!error_message.empty()) {
2481 communication_state_->setPartnerUnavailable();
2482
2483 } else if (last_lease_in_callback) {
2484 // This indicates that there are more leases to be fetched.
2485 // Therefore, we have to send another leaseX-get-page command.
2486 asyncSyncLeases(http_client, remote_config, max_period, last_lease_in_callback,
2487 post_sync_action, dhcp_disabled);
2488 return;
2489 }
2490
2491 // Invoke post synchronization action if it was specified.
2492 if (post_sync_action) {
2493 post_sync_action(error_message.empty(),
2494 error_message,
2495 dhcp_disabled);
2496 }
2497 },
2498 HttpClient::RequestTimeout(config_->getSyncTimeout()),
2499 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
2500 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
2501 std::bind(&HAService::clientCloseHandler, this, ph::_1)
2502 );
2503
2504}
2505
2507HAService::processSynchronize(const std::string& server_name,
2508 const unsigned int max_period) {
2509 HAConfig::PeerConfigPtr remote_config;
2510 try {
2511 remote_config = config_->getPeerConfig(server_name);
2512 } catch (const std::exception& ex) {
2513 return (createAnswer(CONTROL_RESULT_ERROR, ex.what()));
2514 }
2515 // We must not synchronize with self.
2516 if (remote_config->getName() == config_->getThisServerName()) {
2517 return (createAnswer(CONTROL_RESULT_ERROR, "'" + remote_config->getName()
2518 + "' points to local server but should point to a partner"));
2519 }
2520 std::string answer_message;
2521 int sync_status = synchronize(answer_message, remote_config, max_period);
2522 return (createAnswer(sync_status, answer_message));
2523}
2524
2525int
2526HAService::synchronize(std::string& status_message,
2527 const HAConfig::PeerConfigPtr& remote_config,
2528 const unsigned int max_period) {
2529 lease_sync_filter_.apply();
2530
2531 IOServicePtr io_service(new IOService());
2532 HttpClient client(io_service, false);
2533
2534 asyncSyncLeases(client, remote_config, max_period, Lease4Ptr(),
2535 [&](const bool success, const std::string& error_message,
2536 const bool dhcp_disabled) {
2537 // If there was a fatal error while fetching the leases, let's
2538 // log an error message so as it can be included in the response
2539 // to the controlling client.
2540 if (!success) {
2541 status_message = error_message;
2542 }
2543
2544 // Whether or not there was an error while fetching the leases,
2545 // we need to re-enable the DHCP service on the peer if the
2546 // DHCP service was disabled in the course of synchronization.
2547 if (dhcp_disabled) {
2548 // If the synchronization was completed successfully let's
2549 // try to send the ha-sync-complete-notify command to the
2550 // partner.
2551 if (success) {
2552 asyncSyncCompleteNotify(client, remote_config,
2553 [&](const bool success_complete_notify,
2554 const std::string& error_message_complete_notify,
2555 const int rcode) {
2556 // This command may not be supported by the partner when it
2557 // runs an older Kea version. In that case, send the dhcp-enable
2558 // command as in previous Kea version.
2560 asyncEnableDHCPService(client, remote_config,
2561 [&](const bool success_enable_dhcp,
2562 const std::string& error_message_enable_dhcp,
2563 const int) {
2564 // It is possible that we have already recorded an error
2565 // message while synchronizing the lease database. Don't
2566 // override the existing error message.
2567 if (!success_enable_dhcp && status_message.empty()) {
2568 status_message = error_message_enable_dhcp;
2569 }
2570
2571 // The synchronization process is completed, so let's break
2572 // the IO service so as we can return the response to the
2573 // controlling client.
2574 io_service->stop();
2575 });
2576
2577 } else {
2578 // ha-sync-complete-notify command was delivered to the partner.
2579 // The synchronization process ends here.
2580 if (!success_complete_notify && status_message.empty()) {
2581 status_message = error_message_complete_notify;
2582 }
2583
2584 io_service->stop();
2585 }
2586 });
2587
2588 } else {
2589 // Synchronization was unsuccessful. Send the dhcp-enable command to
2590 // re-enable the DHCP service. Note, that we don't send the
2591 // ha-sync-complete-notify command in this case. It is only sent in
2592 // the case when synchronization ends successfully.
2593 asyncEnableDHCPService(client, remote_config,
2594 [&](const bool success_enable_dhcp,
2595 const std::string& error_message_enable_dhcp,
2596 const int) {
2597 if (!success_enable_dhcp && status_message.empty()) {
2598 status_message = error_message_enable_dhcp;
2599 }
2600
2601 // The synchronization process is completed, so let's break
2602 // the IO service so as we can return the response to the
2603 // controlling client.
2604 io_service->stop();
2605
2606 });
2607 }
2608
2609 } else {
2610 // Also stop IO service if there is no need to enable DHCP
2611 // service.
2612 io_service->stop();
2613 }
2614 });
2615
2617 .arg(config_->getThisServerName())
2618 .arg(remote_config->getLogLabel());
2619
2620 // Measure duration of the synchronization.
2621 Stopwatch stopwatch;
2622
2623 // Run the IO service until it is stopped by any of the callbacks. This
2624 // makes it synchronous.
2625 io_service->run();
2626
2627 // End measuring duration.
2628 stopwatch.stop();
2629
2630 client.stop();
2631
2632 io_service->stopAndPoll();
2633
2634 // If an error message has been recorded, return an error to the controlling
2635 // client.
2636 if (!status_message.empty()) {
2638
2640 .arg(config_->getThisServerName())
2641 .arg(remote_config->getLogLabel())
2642 .arg(status_message);
2643
2644 return (CONTROL_RESULT_ERROR);
2645
2646 }
2647
2648 // Everything was fine, so let's return a success.
2649 status_message = "Lease database synchronization complete.";
2651
2653 .arg(config_->getThisServerName())
2654 .arg(remote_config->getLogLabel())
2655 .arg(stopwatch.logFormatLastDuration());
2656
2657 return (CONTROL_RESULT_SUCCESS);
2658}
2659
2660void
2663 PostRequestCallback post_request_action) {
2664 if (lease_update_backlog_.size() == 0) {
2665 post_request_action(true, "", CONTROL_RESULT_SUCCESS);
2666 return;
2667 }
2668
2669 ConstElementPtr command;
2672 Lease4Ptr lease = boost::dynamic_pointer_cast<Lease4>(lease_update_backlog_.pop(op_type));
2673 if (op_type == LeaseUpdateBacklog::ADD) {
2674 command = CommandCreator::createLease4Update(*lease);
2675 } else {
2676 command = CommandCreator::createLease4Delete(*lease);
2677 }
2678
2679 } else {
2681 }
2682
2683 // Create HTTP/1.1 request including our command.
2684 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2686 HostHttpHeader(config->getUrl().getStrippedHostname()));
2687 config->addBasicAuthHttpHeader(request);
2688 request->setBodyAsJson(command);
2689 request->finalize();
2690
2691 // Response object should also be created because the HTTP client needs
2692 // to know the type of the expected response.
2693 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2694
2695 http_client.asyncSendRequest(config->getUrl(), config->getTlsContext(),
2696 request, response,
2697 [this, &http_client, config, post_request_action]
2698 (const boost::system::error_code& ec,
2700 const std::string& error_str) {
2701
2702 int rcode = 0;
2703 std::string error_message;
2704
2705 if (ec || !error_str.empty()) {
2706 error_message = (ec ? ec.message() : error_str);
2707 LOG_WARN(ha_logger, HA_LEASES_BACKLOG_COMMUNICATIONS_FAILED)
2708 .arg(config_->getThisServerName())
2709 .arg(config->getLogLabel())
2710 .arg(ec ? ec.message() : error_str);
2711
2712 } else {
2713 // Handle third group of errors.
2714 try {
2715 auto args = verifyAsyncResponse(http_response, rcode);
2716 } catch (const std::exception& ex) {
2717 error_message = ex.what();
2719 .arg(config_->getThisServerName())
2720 .arg(config->getLogLabel())
2721 .arg(ex.what());
2722 }
2723 }
2724
2725 // Recursively send all outstanding lease updates or break when an
2726 // error occurs. In DHCPv6, this is a single iteration because we use
2727 // lease6-bulk-apply, which combines many lease updates in a single
2728 // transaction. In the case of DHCPv4, each update is sent in its own
2729 // transaction.
2730 if (error_message.empty()) {
2731 asyncSendLeaseUpdatesFromBacklog(http_client, config, post_request_action);
2732 } else {
2733 post_request_action(error_message.empty(), error_message, rcode);
2734 }
2735 });
2736}
2737
2738bool
2740 auto num_updates = lease_update_backlog_.size();
2741 if (num_updates == 0) {
2743 .arg(config_->getThisServerName());
2744 return (true);
2745 }
2746
2747 IOServicePtr io_service(new IOService());
2748 HttpClient client(io_service, false);
2749 auto remote_config = config_->getFailoverPeerConfig();
2750 bool updates_successful = true;
2751
2753 .arg(config_->getThisServerName())
2754 .arg(num_updates)
2755 .arg(remote_config->getName());
2756
2757 asyncSendLeaseUpdatesFromBacklog(client, remote_config,
2758 [&](const bool success, const std::string&, const int) {
2759 io_service->stop();
2760 updates_successful = success;
2761 });
2762
2763 // Measure duration of the updates.
2764 Stopwatch stopwatch;
2765
2766 // Run the IO service until it is stopped by the callback. This makes it synchronous.
2767 io_service->run();
2768
2769 // End measuring duration.
2770 stopwatch.stop();
2771
2772 client.stop();
2773
2774 io_service->stopAndPoll();
2775
2776 if (updates_successful) {
2778 .arg(config_->getThisServerName())
2779 .arg(remote_config->getName())
2780 .arg(stopwatch.logFormatLastDuration());
2781 }
2782
2783 return (updates_successful);
2784}
2785
2786void
2789 PostRequestCallback post_request_action) {
2790 ConstElementPtr command = CommandCreator::createHAReset(config_->getThisServerName(),
2791 server_type_);
2792
2793 // Create HTTP/1.1 request including our command.
2794 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2796 HostHttpHeader(config->getUrl().getStrippedHostname()));
2797 config->addBasicAuthHttpHeader(request);
2798 request->setBodyAsJson(command);
2799 request->finalize();
2800
2801 // Response object should also be created because the HTTP client needs
2802 // to know the type of the expected response.
2803 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2804
2805 http_client.asyncSendRequest(config->getUrl(), config->getTlsContext(),
2806 request, response,
2807 [this, config, post_request_action]
2808 (const boost::system::error_code& ec,
2810 const std::string& error_str) {
2811
2812 int rcode = 0;
2813 std::string error_message;
2814
2815 if (ec || !error_str.empty()) {
2816 error_message = (ec ? ec.message() : error_str);
2817 LOG_WARN(ha_logger, HA_RESET_COMMUNICATIONS_FAILED)
2818 .arg(config_->getThisServerName())
2819 .arg(config->getLogLabel())
2820 .arg(ec ? ec.message() : error_str);
2821
2822 } else {
2823 // Handle third group of errors.
2824 try {
2825 auto args = verifyAsyncResponse(http_response, rcode);
2826 } catch (const std::exception& ex) {
2827 error_message = ex.what();
2829 .arg(config_->getThisServerName())
2830 .arg(config->getLogLabel())
2831 .arg(ex.what());
2832 }
2833 }
2834
2835 post_request_action(error_message.empty(), error_message, rcode);
2836 });
2837}
2838
2839bool
2841 IOServicePtr io_service(new IOService());
2842 HttpClient client(io_service, false);
2843 auto remote_config = config_->getFailoverPeerConfig();
2844 bool reset_successful = true;
2845
2846 asyncSendHAReset(client, remote_config,
2847 [&](const bool success, const std::string&, const int) {
2848 io_service->stop();
2849 reset_successful = success;
2850 });
2851
2852 // Run the IO service until it is stopped by the callback. This makes it synchronous.
2853 io_service->run();
2854
2855 client.stop();
2856
2857 io_service->stopAndPoll();
2858
2859 return (reset_successful);
2860}
2861
2863HAService::processScopes(const std::vector<std::string>& scopes) {
2864 try {
2865 query_filter_.serveScopes(scopes);
2867
2868 } catch (const std::exception& ex) {
2869 return (createAnswer(CONTROL_RESULT_ERROR, ex.what()));
2870 }
2871
2872 return (createAnswer(CONTROL_RESULT_SUCCESS, "New HA scopes configured."));
2873}
2874
2877 if (unpause()) {
2878 return (createAnswer(CONTROL_RESULT_SUCCESS, "HA state machine continues."));
2879 }
2880 return (createAnswer(CONTROL_RESULT_SUCCESS, "HA state machine is not paused."));
2881}
2882
2884HAService::processMaintenanceNotify(const bool cancel, const std::string& state) {
2885 if (cancel) {
2887 return (createAnswer(CONTROL_RESULT_ERROR, "Unable to cancel the"
2888 " maintenance for the server not in the"
2889 " in-maintenance state."));
2890 }
2891
2892 try {
2893 communication_state_->setPartnerState(state);
2894
2895 } catch (...) {
2896 // Hopefully the received state is correct. If it isn't, let's set the
2897 // partner state to unavailable and count on the state machine to resolve.
2898 communication_state_->setPartnerUnavailable();
2899 }
2901 // In rare cases the previous state may be the server's current state. Transitioning
2902 // to it would cause a deadlock and the server will remain stuck in maintenance.
2903 // In these cases let's simply transition to the waiting state and the state machine
2904 // should solve it.
2907
2908 // Communicate the new state to the partner.
2909 ElementPtr arguments = Element::createMap();
2910 std::string state_label = getState(getCurrState())->getLabel();
2911 arguments->set("state", Element::create(state_label));
2912
2913 return (createAnswer(CONTROL_RESULT_SUCCESS, "Server maintenance canceled.", arguments));
2914 }
2915
2916 switch (getCurrState()) {
2917 case HA_BACKUP_ST:
2919 case HA_TERMINATED_ST:
2920 // The reason why we don't return an error result here is that we have to
2921 // have a way to distinguish between the errors caused by the communication
2922 // issues and the cases when there is no communication error but the server
2923 // is not allowed to enter the in-maintenance state. In the former case, the
2924 // partner would go to partner-down. In the case signaled by the special
2925 // result code entering the maintenance state is not allowed.
2927 "Unable to transition the server from the "
2928 + stateToString(getCurrState()) + " to"
2929 " in-maintenance state."));
2930 default:
2933 }
2934 return (createAnswer(CONTROL_RESULT_SUCCESS, "Server is in-maintenance state."));
2935}
2936
2939 switch (getCurrState()) {
2940 case HA_BACKUP_ST:
2943 case HA_DOWN_ST:
2944 case HA_TERMINATED_ST:
2945 return (createAnswer(CONTROL_RESULT_ERROR, "Unable to transition the server from"
2946 " the " + stateToString(getCurrState()) + " to"
2947 " partner-in-maintenance state."));
2948 default:
2949 ;
2950 }
2951
2952 HAConfig::PeerConfigPtr remote_config = config_->getFailoverPeerConfig();
2953
2954 // Create HTTP/1.1 request including ha-maintenance-notify command
2955 // with the cancel flag set to false.
2956 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2958 HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
2959 remote_config->addBasicAuthHttpHeader(request);
2960 request->setBodyAsJson(CommandCreator::createMaintenanceNotify(config_->getThisServerName(),
2961 false, getCurrState(), server_type_));
2962 request->finalize();
2963
2964 // Response object should also be created because the HTTP client needs
2965 // to know the type of the expected response.
2966 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2967
2968 IOServicePtr io_service(new IOService());
2969 HttpClient client(io_service, false);
2970
2971 boost::system::error_code captured_ec;
2972 std::string captured_error_message;
2973 int captured_rcode = 0;
2974
2975 // Schedule asynchronous HTTP request.
2976 client.asyncSendRequest(remote_config->getUrl(),
2977 remote_config->getTlsContext(),
2978 request, response,
2979 [this, remote_config, &io_service, &captured_ec, &captured_error_message,
2980 &captured_rcode]
2981 (const boost::system::error_code& ec,
2983 const std::string& error_str) {
2984
2985 io_service->stop();
2986
2987 // There are three possible groups of errors. One is the IO error
2988 // causing issues in communication with the peer. Another one is
2989 // an HTTP parsing error. The last type of error is when non-success
2990 // error code is returned in the response carried in the HTTP message
2991 // or if the JSON response is otherwise broken.
2992
2993 std::string error_message;
2994
2995 // Handle first two groups of errors.
2996 if (ec || !error_str.empty()) {
2997 error_message = (ec ? ec.message() : error_str);
2998 LOG_ERROR(ha_logger, HA_MAINTENANCE_NOTIFY_COMMUNICATIONS_FAILED)
2999 .arg(config_->getThisServerName())
3000 .arg(remote_config->getLogLabel())
3001 .arg(error_message);
3002
3003 } else {
3004
3005 // Handle third group of errors.
3006 try {
3007 static_cast<void>(verifyAsyncResponse(http_response, captured_rcode));
3008
3009 } catch (const std::exception& ex) {
3010 error_message = ex.what();
3012 .arg(config_->getThisServerName())
3013 .arg(remote_config->getLogLabel())
3014 .arg(error_message);
3015 }
3016 }
3017
3018 // If there was an error communicating with the partner, mark the
3019 // partner as unavailable.
3020 if (!error_message.empty()) {
3021 communication_state_->setPartnerUnavailable();
3022 }
3023
3024 captured_ec = ec;
3025 captured_error_message = error_message;
3026 },
3028 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
3029 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
3030 std::bind(&HAService::clientCloseHandler, this, ph::_1)
3031 );
3032
3033 // Run the IO service until it is stopped by any of the callbacks. This
3034 // makes it synchronous.
3035 io_service->run();
3036
3037 client.stop();
3038
3039 io_service->stopAndPoll();
3040
3041 // If there was a communication problem with the partner we assume that
3042 // the partner is already down while we receive this command.
3043 if (captured_ec || (captured_rcode == CONTROL_RESULT_ERROR)) {
3044 postNextEvent(HA_MAINTENANCE_START_EVT);
3045 verboseTransition(HA_PARTNER_DOWN_ST);
3046 runModel(NOP_EVT);
3048 "Server is now in the partner-down state as its"
3049 " partner appears to be offline for maintenance."));
3050
3051 } else if (captured_rcode == CONTROL_RESULT_SUCCESS) {
3052 // If the partner responded indicating no error it means that the
3053 // partner has been transitioned to the in-maintenance state. In that
3054 // case we transition to the partner-in-maintenance state.
3055 postNextEvent(HA_MAINTENANCE_START_EVT);
3056 verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST);
3057 runModel(NOP_EVT);
3058
3059 } else {
3060 // Partner server returned a special status code which means that it can't
3061 // transition to the partner-in-maintenance state.
3062 return (createAnswer(CONTROL_RESULT_ERROR, "Unable to transition to the"
3063 " partner-in-maintenance state. The partner server responded"
3064 " with the following message to the ha-maintenance-notify"
3065 " command: " + captured_error_message + "."));
3066
3067 }
3068
3070 "Server is now in the partner-in-maintenance state"
3071 " and its partner is in-maintenance state. The partner"
3072 " can be now safely shut down."));
3073}
3074
3078 return (createAnswer(CONTROL_RESULT_ERROR, "Unable to cancel maintenance"
3079 " request because the server is not in the"
3080 " partner-in-maintenance state."));
3081 }
3082
3083 // This is the state the server will transition to if the notification to the
3084 // partner is successful.
3086
3087 HAConfig::PeerConfigPtr remote_config = config_->getFailoverPeerConfig();
3088
3089 // Create HTTP/1.1 request including ha-maintenance-notify command
3090 // with the cancel flag set to true.
3091 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
3093 HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
3094 remote_config->addBasicAuthHttpHeader(request);
3095 request->setBodyAsJson(CommandCreator::createMaintenanceNotify(config_->getThisServerName(),
3096 true,
3097 next_state,
3098 server_type_));
3099 request->finalize();
3100
3101 // Response object should also be created because the HTTP client needs
3102 // to know the type of the expected response.
3103 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
3104
3105 IOServicePtr io_service(new IOService());
3106 HttpClient client(io_service, false);
3107
3108 std::string error_message;
3109
3110 // Schedule asynchronous HTTP request.
3111 client.asyncSendRequest(remote_config->getUrl(),
3112 remote_config->getTlsContext(),
3113 request, response,
3114 [this, remote_config, &io_service, &error_message]
3115 (const boost::system::error_code& ec,
3117 const std::string& error_str) {
3118
3119 io_service->stop();
3120
3121 // Handle first two groups of errors.
3122 if (ec || !error_str.empty()) {
3123 error_message = (ec ? ec.message() : error_str);
3124 LOG_ERROR(ha_logger, HA_MAINTENANCE_NOTIFY_CANCEL_COMMUNICATIONS_FAILED)
3125 .arg(config_->getThisServerName())
3126 .arg(remote_config->getLogLabel())
3127 .arg(error_message);
3128
3129 } else {
3130
3131 // Handle third group of errors.
3132 try {
3133 int rcode = 0;
3134 ConstElementPtr args = verifyAsyncResponse(http_response, rcode);
3135
3136 // Partner's state has changed after the notification. However, we don't know
3137 // its new state. We'll check if the partner returned its state. If it didn't,
3138 // we set the unavailable state as a default.
3139 communication_state_->setPartnerUnavailable();
3140
3141 // Newer Kea versions return the state of the notified server.
3142 // Older versions don't, so the arguments may not be present.
3143 if (args && args->getType() == Element::map) {
3144 // Arguments may include partner's state.
3145 ConstElementPtr state = args->get("state");
3146 if (state) {
3147 if (state->getType() != Element::string) {
3148 isc_throw(CtrlChannelError, "server state not returned in response"
3149 " to a ha-heartbeat command or it is not a string");
3150 }
3151 communication_state_->setPartnerState(state->stringValue());
3152 }
3153 }
3154 } catch (const std::exception& ex) {
3155 error_message = ex.what();
3157 .arg(config_->getThisServerName())
3158 .arg(remote_config->getLogLabel())
3159 .arg(error_message);
3160 }
3161 }
3162
3163 // If there was an error communicating with the partner, mark the
3164 // partner as unavailable.
3165 if (!error_message.empty()) {
3166 communication_state_->setPartnerUnavailable();
3167 }
3168 },
3170 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
3171 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
3172 std::bind(&HAService::clientCloseHandler, this, ph::_1)
3173 );
3174
3175 // Run the IO service until it is stopped by any of the callbacks. This
3176 // makes it synchronous.
3177 io_service->run();
3178
3179 client.stop();
3180
3181 io_service->stopAndPoll();
3182
3183 // There was an error in communication with the partner or the
3184 // partner was unable to revert its state.
3185 if (!error_message.empty()) {
3187 "Unable to cancel maintenance. The partner server responded"
3188 " with the following message to the ha-maintenance-notify"
3189 " command: " + error_message + "."));
3190 }
3191
3192 // Successfully reverted partner's state. Let's also revert our state to the
3193 // previous one. Avoid returning to the partner-in-maintenance if it was
3194 // the previous state.
3195 postNextEvent(HA_MAINTENANCE_CANCEL_EVT);
3196 verboseTransition(next_state);
3197 runModel(NOP_EVT);
3198
3200 "Server maintenance successfully canceled."));
3201}
3202
3203void
3205 const HAConfig::PeerConfigPtr& remote_config,
3206 PostRequestCallback post_request_action) {
3207 // Create HTTP/1.1 request including our command.
3208 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
3210 HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
3211
3212 remote_config->addBasicAuthHttpHeader(request);
3213 request->setBodyAsJson(CommandCreator::createSyncCompleteNotify(getRemoteOrigin(),
3214 config_->getThisServerName(),
3215 server_type_));
3216 request->finalize();
3217
3218 // Response object should also be created because the HTTP client needs
3219 // to know the type of the expected response.
3220 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
3221
3222 // Schedule asynchronous HTTP request.
3223 http_client.asyncSendRequest(remote_config->getUrl(),
3224 remote_config->getTlsContext(),
3225 request, response,
3226 [this, remote_config, post_request_action]
3227 (const boost::system::error_code& ec,
3229 const std::string& error_str) {
3230
3231 // There are three possible groups of errors. One is the IO error
3232 // causing issues in communication with the peer. Another one is an
3233 // HTTP parsing error. The last type of error is when non-success
3234 // error code is returned in the response carried in the HTTP message
3235 // or if the JSON response is otherwise broken.
3236
3237 int rcode = 0;
3238 std::string error_message;
3239
3240 // Handle first two groups of errors.
3241 if (ec || !error_str.empty()) {
3242 error_message = (ec ? ec.message() : error_str);
3243 LOG_ERROR(ha_logger, HA_SYNC_COMPLETE_NOTIFY_COMMUNICATIONS_FAILED)
3244 .arg(config_->getThisServerName())
3245 .arg(remote_config->getLogLabel())
3246 .arg(error_message);
3247
3248 } else {
3249
3250 // Handle third group of errors.
3251 try {
3252 static_cast<void>(verifyAsyncResponse(http_response, rcode));
3253
3254 } catch (const CommandUnsupportedError& ex) {
3256
3257 } catch (const std::exception& ex) {
3258 error_message = ex.what();
3260 .arg(config_->getThisServerName())
3261 .arg(remote_config->getLogLabel())
3262 .arg(error_message);
3263 }
3264 }
3265
3266 // If there was an error communicating with the partner, mark the
3267 // partner as unavailable.
3268 if (!error_message.empty()) {
3269 communication_state_->setPartnerUnavailable();
3270 }
3271
3272 // Invoke post request action if it was specified.
3273 if (post_request_action) {
3274 post_request_action(error_message.empty(),
3275 error_message,
3276 rcode);
3277 }
3278 },
3280 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
3281 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
3282 std::bind(&HAService::clientCloseHandler, this, ph::_1)
3283 );
3284}
3285
3287HAService::processSyncCompleteNotify(const unsigned int origin_id) {
3290 // We're in the partner-down state and the partner notified us
3291 // that it has synchronized its database. We can't enable the
3292 // service yet, because it may result in some new lease allocations
3293 // that the partner would miss (we don't send lease updates in the
3294 // partner-down state). We must first send the heartbeat and let
3295 // the state machine resolve the situation between the partners.
3296 // It may unblock the network service.
3297 network_state_->disableService(getLocalOrigin());
3298 }
3299 // Release the network state lock for the remote origin because we have
3300 // acquired the local network state lock above (partner-down state), or
3301 // we don't need the lock (other states).
3302 network_state_->enableService(origin_id);
3304 "Server successfully notified about the synchronization completion."));
3305}
3306
3309 // Set the return code to error in case of early throw.
3310 rcode = CONTROL_RESULT_ERROR;
3311 // The response must cast to JSON type.
3312 HttpResponseJsonPtr json_response =
3313 boost::dynamic_pointer_cast<HttpResponseJson>(response);
3314 if (!json_response) {
3315 isc_throw(CtrlChannelError, "no valid HTTP response found");
3316 }
3317
3318 // Body holds the response to our command.
3319 ConstElementPtr body = json_response->getBodyAsJson();
3320 if (!body) {
3321 isc_throw(CtrlChannelError, "no body found in the response");
3322 }
3323
3324 // Body should contain a list of responses from multiple servers.
3325 if (body->getType() != Element::list) {
3326 // Some control socket errors are returned as a map.
3327 if (body->getType() == Element::map) {
3329 ElementPtr answer = Element::createMap();
3330 answer->set(CONTROL_RESULT, Element::create(rcode));
3331 ConstElementPtr text = body->get(CONTROL_TEXT);
3332 if (text) {
3333 answer->set(CONTROL_TEXT, text);
3334 }
3335 list->add(answer);
3336 body = list;
3337 } else {
3338 isc_throw(CtrlChannelError, "body of the response must be a list");
3339 }
3340 }
3341
3342 // There must be at least one response.
3343 if (body->empty()) {
3344 isc_throw(CtrlChannelError, "list of responses must not be empty");
3345 }
3346
3347 // Check if the status code of the first response. We don't support multiple
3348 // at this time, because we always send a request to a single location.
3349 ConstElementPtr args = parseAnswer(rcode, body->get(0));
3350 if (rcode == CONTROL_RESULT_SUCCESS) {
3351 return (args);
3352 }
3353
3354 std::ostringstream s;
3355
3356 // The empty status can occur for the lease6-bulk-apply command. In that
3357 // case, the response may contain conflicted or erred leases within the
3358 // arguments, rather than globally. For other error cases let's construct
3359 // the error message from the global values.
3360 if (rcode != CONTROL_RESULT_EMPTY) {
3361 // Include an error text if available.
3362 if (args && args->getType() == Element::string) {
3363 s << args->stringValue() << " (";
3364 }
3365 // Include an error code.
3366 s << "error code " << rcode << ")";
3367 }
3368
3369 switch (rcode) {
3371 isc_throw(CommandUnsupportedError, s.str());
3372
3374 isc_throw(ConflictError, s.str());
3375
3377 // Handle the lease6-bulk-apply error cases.
3378 if (args && (args->getType() == Element::map)) {
3379 auto failed_leases = args->get("failed-leases");
3380 if (!failed_leases || (failed_leases->getType() != Element::list)) {
3381 // If there are no failed leases there is nothing to do.
3382 break;
3383 }
3384 auto conflict = false;
3385 ConstElementPtr conflict_error_message;
3386 for (unsigned i = 0; i < failed_leases->size(); ++i) {
3387 auto lease = failed_leases->get(i);
3388 if (!lease || lease->getType() != Element::map) {
3389 continue;
3390 }
3391 auto result = lease->get("result");
3392 if (!result || result->getType() != Element::integer) {
3393 continue;
3394 }
3395 auto error_message = lease->get("error-message");
3396 // Error status code takes precedence over the conflict.
3397 if (result->intValue() == CONTROL_RESULT_ERROR) {
3398 if (error_message && error_message->getType()) {
3399 s << error_message->stringValue() << " (";
3400 }
3401 s << "error code " << result->intValue() << ")";
3402 isc_throw(CtrlChannelError, s.str());
3403 }
3404 if (result->intValue() == CONTROL_RESULT_CONFLICT) {
3405 // Let's record the conflict but there may still be some
3406 // leases with an error status code, so do not throw the
3407 // conflict exception yet.
3408 conflict = true;
3409 conflict_error_message = error_message;
3410 }
3411 }
3412 if (conflict) {
3413 // There are no errors. There are only conflicts. Throw
3414 // appropriate exception.
3415 if (conflict_error_message &&
3416 (conflict_error_message->getType() == Element::string)) {
3417 s << conflict_error_message->stringValue() << " (";
3418 }
3419 s << "error code " << CONTROL_RESULT_CONFLICT << ")";
3420 isc_throw(ConflictError, s.str());
3421 }
3422 }
3423 break;
3424 default:
3425 isc_throw(CtrlChannelError, s.str());
3426 }
3427 return (args);
3428}
3429
3430bool
3431HAService::clientConnectHandler(const boost::system::error_code& ec, int tcp_native_fd) {
3432
3433 // If client is running it's own IOService we do NOT want to
3434 // register the socket with IfaceMgr.
3435 if (client_->getThreadIOService()) {
3436 return (true);
3437 }
3438
3439 // If things look OK register the socket with Interface Manager. Note
3440 // we don't register if the FD is < 0 to avoid an exception throw.
3441 // It is unlikely that this will occur but we want to be liberal
3442 // and avoid issues.
3443 if ((!ec || (ec.value() == boost::asio::error::in_progress))
3444 && (tcp_native_fd >= 0)) {
3445 // External socket callback is a NOP. Ready events handlers are
3446 // run by an explicit call IOService ready in kea-dhcp<n> code.
3447 // We are registering the socket only to interrupt main-thread
3448 // select().
3449 IfaceMgr::instance().addExternalSocket(tcp_native_fd,
3450 std::bind(&HAService::socketReadyHandler, this, ph::_1)
3451 );
3452 }
3453
3454 // If ec.value() == boost::asio::error::already_connected, we should already
3455 // be registered, so nothing to do. If it is any other value, then connect
3456 // failed and Connection logic should handle that, not us, so no matter
3457 // what happens we're returning true.
3458 return (true);
3459}
3460
3461void
3463 // If the socket is ready but does not belong to one of our client's
3464 // ongoing transactions, we close it. This will unregister it from
3465 // IfaceMgr and ensure the client starts over with a fresh connection
3466 // if it needs to do so.
3467 client_->closeIfOutOfBand(tcp_native_fd);
3468}
3469
3470void
3472 if ((tcp_native_fd >= 0) &&
3473 IfaceMgr::instance().isExternalSocket(tcp_native_fd)) {
3475 }
3476}
3477
3478size_t
3480 if (MultiThreadingMgr::instance().getMode()) {
3481 std::lock_guard<std::mutex> lock(mutex_);
3482 return (pending_requests_.size());
3483 } else {
3484 return (pending_requests_.size());
3485 }
3486}
3487
3488template<typename QueryPtrType>
3489int
3490HAService::getPendingRequest(const QueryPtrType& query) {
3491 if (MultiThreadingMgr::instance().getMode()) {
3492 std::lock_guard<std::mutex> lock(mutex_);
3493 return (getPendingRequestInternal(query));
3494 } else {
3495 return (getPendingRequestInternal(query));
3496 }
3497}
3498
3499template<typename QueryPtrType>
3500int
3501HAService::getPendingRequestInternal(const QueryPtrType& query) {
3502 if (pending_requests_.count(query) == 0) {
3503 return (0);
3504 } else {
3505 return (pending_requests_[query]);
3506 }
3507}
3508
3509void
3511 // Since this function is used as CS callback all exceptions must be
3512 // suppressed (except the @ref MultiThreadingInvalidOperation), unlikely
3513 // though they may be.
3514 // The @ref MultiThreadingInvalidOperation is propagated to the scope of the
3515 // @ref MultiThreadingCriticalSection constructor.
3516 try {
3517 if (client_) {
3518 client_->checkPermissions();
3519 }
3520
3521 if (listener_) {
3522 listener_->checkPermissions();
3523 }
3524 } catch (const isc::MultiThreadingInvalidOperation& ex) {
3526 .arg(config_->getThisServerName())
3527 .arg(ex.what());
3528 // The exception needs to be propagated to the caller of the
3529 // @ref MultiThreadingCriticalSection constructor.
3530 throw;
3531 } catch (const std::exception& ex) {
3533 .arg(config_->getThisServerName())
3534 .arg(ex.what());
3535 }
3536}
3537
3538void
3540 // Add critical section callbacks.
3543 std::bind(&HAService::pauseClientAndListener, this),
3544 std::bind(&HAService::resumeClientAndListener, this));
3545
3546 if (client_) {
3547 client_->start();
3548 }
3549
3550 if (listener_) {
3551 listener_->start();
3552 }
3553}
3554
3555void
3557 // Since this function is used as CS callback all exceptions must be
3558 // suppressed, unlikely though they may be.
3559 try {
3560 if (client_) {
3561 client_->pause();
3562 }
3563
3564 if (listener_) {
3565 listener_->pause();
3566 }
3567 } catch (const std::exception& ex) {
3569 .arg(config_->getThisServerName())
3570 .arg(ex.what());
3571 }
3572}
3573
3574void
3576 // Since this function is used as CS callback all exceptions must be
3577 // suppressed, unlikely though they may be.
3578 try {
3579 if (client_) {
3580 client_->resume();
3581 }
3582
3583 if (listener_) {
3584 listener_->resume();
3585 }
3586 } catch (std::exception& ex) {
3588 .arg(config_->getThisServerName())
3589 .arg(ex.what());
3590 }
3591}
3592
3593void
3595 // Remove critical section callbacks.
3597
3598 if (client_) {
3599 client_->stop();
3600 }
3601
3602 if (listener_) {
3603 listener_->stop();
3604 }
3605}
3606
3607// Explicit instantiations.
3608template int HAService::getPendingRequest(const Pkt4Ptr&);
3609template int HAService::getPendingRequest(const Pkt6Ptr&);
3610
3611} // end of namespace isc::ha
3612} // end of namespace isc
static ElementPtr create(const Position &pos=ZERO_POSITION())
Create a NullElement.
Definition data.cc:300
@ map
Definition data.h:160
@ integer
Definition data.h:153
@ list
Definition data.h:159
@ string
Definition data.h:157
static ElementPtr createMap(const Position &pos=ZERO_POSITION())
Creates an empty MapElement type ElementPtr.
Definition data.cc:355
static ElementPtr createList(const Position &pos=ZERO_POSITION())
Creates an empty ListElement type ElementPtr.
Definition data.cc:350
virtual const char * what() const
Returns a C-style character string of the cause of the exception.
Exception thrown when a worker thread is trying to stop or pause the respective thread pool (which wo...
A generic exception that is thrown when an unexpected error condition occurs.
A multi-threaded HTTP listener that can process API commands requests.
A standard control channel exception that is thrown if a function is there is a problem with one of t...
void deleteExternalSocket(int socketfd)
Deletes external socket.
Definition iface_mgr.cc:398
static IfaceMgr & instance()
IfaceMgr is a singleton class.
Definition iface_mgr.cc:52
void addExternalSocket(int socketfd, SocketCallback callback)
Adds external socket and a callback.
Definition iface_mgr.cc:367
static data::ConstElementPtr createLease4Delete(const dhcp::Lease4 &lease4)
Creates lease4-del command.
static data::ConstElementPtr createHeartbeat(const std::string &server_name, const HAServerType &server_type)
Creates ha-heartbeat command for DHCP server.
static std::unordered_set< std::string > ha_commands4_
List of commands used by the High Availability in v4.
static data::ConstElementPtr createLease4Update(const dhcp::Lease4 &lease4)
Creates lease4-update command.
static data::ConstElementPtr createSyncCompleteNotify(const unsigned int origin_id, const std::string &server_name, const HAServerType &server_type)
Creates ha-sync-complete-notify command.
static data::ConstElementPtr createLease6BulkApply(const dhcp::Lease6CollectionPtr &leases, const dhcp::Lease6CollectionPtr &deleted_leases)
Creates lease6-bulk-apply command.
static data::ConstElementPtr createLease6GetPage(const dhcp::Lease6Ptr &lease6, const uint32_t limit)
Creates lease6-get-page command.
static data::ConstElementPtr createDHCPDisable(const unsigned int origin_id, const unsigned int max_period, const HAServerType &server_type)
Creates dhcp-disable command for DHCP server.
static data::ConstElementPtr createDHCPEnable(const unsigned int origin_id, const HAServerType &server_type)
Creates dhcp-enable command for DHCP server.
static data::ConstElementPtr createMaintenanceNotify(const std::string &server_name, const bool cancel, const int state, const HAServerType &server_type)
Creates ha-maintenance-notify command.
static std::unordered_set< std::string > ha_commands6_
List of commands used by the High Availability in v6.
static data::ConstElementPtr createHAReset(const std::string &server_name, const HAServerType &server_type)
Creates ha-reset command.
static data::ConstElementPtr createLease4GetPage(const dhcp::Lease4Ptr &lease4, const uint32_t limit)
Creates lease4-get-page command.
Holds communication state between DHCPv4 servers.
Holds communication state between DHCPv6 servers.
static std::string roleToString(const HAConfig::PeerConfig::Role &role)
Returns role name.
Definition ha_config.cc:85
Role
Server's role in the High Availability setup.
Definition ha_config.h:85
std::map< std::string, PeerConfigPtr > PeerConfigMap
Map of the servers' configurations.
Definition ha_config.h:258
static std::string HAModeToString(const HAMode &ha_mode)
Returns HA mode name.
Definition ha_config.cc:236
boost::shared_ptr< PeerConfig > PeerConfigPtr
Pointer to the server's configuration.
Definition ha_config.h:255
static const int HA_MAINTENANCE_START_EVT
ha-maintenance-start command received.
Definition ha_service.h:72
bool inScope(dhcp::Pkt4Ptr &query4)
Checks if the DHCPv4 query should be processed by this server.
void adjustNetworkState()
Enables or disables network state depending on the served scopes.
void stopClientAndListener()
Stop the client and(or) listener instances.
int getNormalState() const
Returns normal operation state for the current configuration.
bool shouldQueueLeaseUpdates(const HAConfig::PeerConfigPtr &peer_config) const
Checks if the lease updates should be queued.
static const int HA_HEARTBEAT_COMPLETE_EVT
Finished heartbeat command.
Definition ha_service.h:57
void asyncSendHAReset(http::HttpClient &http_client, const HAConfig::PeerConfigPtr &remote_config, PostRequestCallback post_request_action)
Sends ha-reset command to partner asynchronously.
bool clientConnectHandler(const boost::system::error_code &ec, int tcp_native_fd)
HttpClient connect callback handler.
void asyncSyncLeases()
Asynchronously reads leases from a peer and updates local lease database.
bool isMaintenanceCanceled() const
Convenience method checking if the current state is a result of canceling the maintenance.
data::ConstElementPtr processMaintenanceCancel()
Processes ha-maintenance-cancel command and returns a response.
void checkPermissionsClientAndListener()
Check client and(or) listener current thread permissions to perform thread pool state transition.
bool shouldReclaim(const dhcp::Lease4Ptr &lease4) const
Checks if the lease should be reclaimed by this server.
void asyncSendLeaseUpdate(const QueryPtrType &query, const HAConfig::PeerConfigPtr &config, const data::ConstElementPtr &command, const hooks::ParkingLotHandlePtr &parking_lot)
Asynchronously sends lease update to the peer.
void verboseTransition(const unsigned state)
Transitions to a desired state and logs it.
bool sendLeaseUpdatesFromBacklog()
Attempts to send all lease updates from the backlog synchronously.
config::CmdHttpListenerPtr listener_
HTTP listener instance used to receive and respond to HA commands and lease updates.
void clientCloseHandler(int tcp_native_fd)
HttpClient close callback handler.
bool leaseUpdateComplete(QueryPtrType &query, const hooks::ParkingLotHandlePtr &parking_lot)
Handle last pending request for this query.
HAConfigPtr config_
Pointer to the HA hooks library configuration.
data::ConstElementPtr processMaintenanceStart()
Processes ha-maintenance-start command and returns a response.
unsigned int id_
Unique service id.
HAServerType server_type_
DHCP server type.
bool sync_complete_notified_
An indicator that a partner sent ha-sync-complete-notify command.
bool shouldTerminate() const
Indicates if the server should transition to the terminated state.
data::ConstElementPtr processScopes(const std::vector< std::string > &scopes)
Processes ha-scopes command and returns a response.
dhcp::NetworkStatePtr network_state_
Pointer to the state of the DHCP service (enabled/disabled).
data::ConstElementPtr processSynchronize(const std::string &server_name, const unsigned int max_period)
Processes ha-sync command and returns a response.
void scheduleHeartbeat()
Schedules asynchronous heartbeat to a peer if it is not scheduled.
void asyncSyncCompleteNotify(http::HttpClient &http_client, const HAConfig::PeerConfigPtr &remote_config, PostRequestCallback post_request_action)
Schedules asynchronous "ha-sync-complete-notify" command to the specified server.
QueryFilter query_filter_
Selects queries to be processed/dropped.
static const int HA_MAINTENANCE_NOTIFY_EVT
ha-maintenance-notify command received.
Definition ha_service.h:69
static const int HA_SYNCED_PARTNER_UNAVAILABLE_EVT
The heartbeat command failed after receiving ha-sync-complete-notify command from the partner.
Definition ha_service.h:79
data::ConstElementPtr processMaintenanceNotify(const bool cancel, const std::string &state)
Processes ha-maintenance-notify command and returns a response.
void conditionalLogPausedState() const
Logs if the server is paused in the current state.
bool unpause()
Unpauses the HA state machine with logging.
static const int HA_CONTROL_RESULT_MAINTENANCE_NOT_ALLOWED
Control result returned in response to ha-maintenance-notify.
Definition ha_service.h:82
void serveDefaultScopes()
Instructs the HA service to serve default scopes.
size_t asyncSendLeaseUpdates(const dhcp::Pkt4Ptr &query, const dhcp::Lease4CollectionPtr &leases, const dhcp::Lease4CollectionPtr &deleted_leases, const hooks::ParkingLotHandlePtr &parking_lot)
Schedules asynchronous IPv4 leases updates.
bool isDHCPServiceDisabled() const
Convenience method checking if the dhcp state has been disabled by user of by database connection.
size_t pendingRequestSize()
Get the number of entries in the pending request map.
static const int HA_SYNCING_SUCCEEDED_EVT
Lease database synchronization succeeded.
Definition ha_service.h:66
bool sendHAReset()
Sends ha-reset command to partner synchronously.
std::function< void(const bool, const std::string &, const int)> PostRequestCallback
Callback invoked when request was sent and a response received or an error occurred.
Definition ha_service.h:96
asiolink::IOServicePtr io_service_
Pointer to the IO service object shared between this hooks library and the DHCP server.
void localDisableDHCPService()
Disables local DHCP service.
CommunicationStatePtr communication_state_
Holds communication state with a peer.
void logFailedLeaseUpdates(const dhcp::PktPtr &query, const data::ConstElementPtr &args) const
Log failed lease updates.
LeaseUpdateBacklog lease_update_backlog_
Backlog of DHCP lease updates.
virtual ~HAService()
Destructor.
static const int HA_SYNCING_FAILED_EVT
Lease database synchronization failed.
Definition ha_service.h:63
static const int HA_MAINTENANCE_CANCEL_EVT
ha-maintenance-cancel command received.
Definition ha_service.h:75
void asyncSendLeaseUpdatesFromBacklog(http::HttpClient &http_client, const HAConfig::PeerConfigPtr &remote_config, PostRequestCallback post_request_action)
Sends lease updates from backlog to partner asynchronously.
data::ConstElementPtr processHeartbeat()
Processes ha-heartbeat command and returns a response.
void asyncSyncLeasesInternal(http::HttpClient &http_client, const HAConfig::PeerConfigPtr &remote_config, const unsigned int max_period, const dhcp::LeasePtr &last_lease, PostSyncCallback post_sync_action, const bool dhcp_disabled)
Implements fetching one page of leases during synchronization.
data::ConstElementPtr processHAReset()
Processes ha-reset command and returns a response.
size_t asyncSendSingleLeaseUpdate(const dhcp::Pkt4Ptr &query, const dhcp::Lease4Ptr &lease, const hooks::ParkingLotHandlePtr &parking_lot)
Schedules an asynchronous IPv4 lease update.
void asyncSendHeartbeat()
Starts asynchronous heartbeat to a peer.
bool isPartnerStateInvalid() const
Indicates if the partner's state is invalid.
void startClientAndListener()
Start the client and(or) listener instances.
data::ConstElementPtr verifyAsyncResponse(const http::HttpResponsePtr &response, int &rcode)
Checks if the response is valid or contains an error.
void resumeClientAndListener()
Resumes client and(or) listener thread pool operations.
data::ConstElementPtr processStatusGet() const
Processes status-get command and returns a response.
int getPendingRequest(const QueryPtrType &query)
Get the number of scheduled requests for a given query.
LeaseSyncFilter lease_sync_filter_
Lease synchronization filter used in hub-and-spoke model.
int synchronize(std::string &status_message, const HAConfig::PeerConfigPtr &remote_config, const unsigned int max_period)
Synchronizes lease database with a partner.
bool shouldSendLeaseUpdates(const HAConfig::PeerConfigPtr &peer_config) const
Checks if the lease updates should be sent as result of leases allocation or release.
void serveFailoverScopes()
Instructs the HA service to serve failover scopes.
void localEnableDHCPService()
Enables local DHCP service.
static const int HA_LEASE_UPDATES_COMPLETE_EVT
Finished lease updates commands.
Definition ha_service.h:60
HAService(const unsigned int id, const asiolink::IOServicePtr &io_service, const dhcp::NetworkStatePtr &network_state, const HAConfigPtr &config, const HAServerType &server_type=HAServerType::DHCPv4)
Constructor.
Definition ha_service.cc:80
void socketReadyHandler(int tcp_native_fd)
IfaceMgr external socket ready callback handler.
http::HttpClientPtr client_
HTTP client instance used to send HA commands and lease updates.
void updatePendingRequest(QueryPtrType &query)
Update pending request counter for this query.
bool shouldPartnerDown() const
Indicates if the server should transition to the partner down state.
void startHeartbeat()
Unconditionally starts one heartbeat to a peer.
data::ConstElementPtr processSyncCompleteNotify(const unsigned int origin_id)
Process ha-sync-complete-notify command and returns a response.
data::ConstElementPtr processContinue()
Processes ha-continue command and returns a response.
void asyncDisableDHCPService(http::HttpClient &http_client, const HAConfig::PeerConfigPtr &remote_config, const unsigned int max_period, PostRequestCallback post_request_action)
Schedules asynchronous "dhcp-disable" command to the specified server.
void pauseClientAndListener()
Pauses client and(or) listener thread pool operations.
std::function< void(const bool, const std::string &, const bool)> PostSyncCallback
Callback invoked when lease database synchronization is complete.
Definition ha_service.h:105
static const int HA_WAITING_TO_TERMINATED_ST_DELAY_MINUTES
A delay in minutes to transition from the waiting to terminated state when the partner remains in ter...
Definition ha_service.h:86
void asyncEnableDHCPService(http::HttpClient &http_client, const HAConfig::PeerConfigPtr &remote_config, PostRequestCallback post_request_action)
Schedules asynchronous "dhcp-enable" command to the specified server.
OpType
Type of the lease update (operation type).
bool inScope(const dhcp::Pkt4Ptr &query4, std::string &scope_class) const
Checks if this server should process the DHCPv4 query.
Represents HTTP Host header.
Definition http_header.h:68
HTTP client class.
void stop()
Halts client-side IO activity.
Definition client.cc:2048
void asyncSendRequest(const Url &url, const asiolink::TlsContextPtr &tls_context, const HttpRequestPtr &request, const HttpResponsePtr &response, const RequestHandler &request_callback, const RequestTimeout &request_timeout=RequestTimeout(10000), const ConnectHandler &connect_callback=ConnectHandler(), const HandshakeHandler &handshake_callback=HandshakeHandler(), const CloseHandler &close_callback=CloseHandler())
Queues new asynchronous HTTP request for a given URL.
Definition client.cc:1987
This class parses and generates time values used in HTTP.
Definition date_time.h:41
std::string rfc1123Format() const
Returns time value formatted as specified in RFC 1123.
Definition date_time.cc:39
static MultiThreadingMgr & instance()
Returns a single instance of Multi Threading Manager.
void removeCriticalSectionCallbacks(const std::string &name)
Removes the set of callbacks associated with a given name from the list of CriticalSection callbacks.
void addCriticalSectionCallbacks(const std::string &name, const CSCallbackSet::Callback &check_cb, const CSCallbackSet::Callback &entry_cb, const CSCallbackSet::Callback &exit_cb)
Adds a set of callbacks to the list of CriticalSection callbacks.
std::string getStateLabel(const int state) const
Fetches the label associated with an state value.
void unpauseModel()
Unpauses state model.
int getLastEvent() const
Fetches the model's last event.
bool isModelPaused() const
Returns whether or not the model is paused.
virtual void defineEvents()
Populates the set of events.
bool doOnExit()
Checks if on exit flag is true.
void defineEvent(int value, const std::string &label)
Adds an event value and associated label to the set of events.
virtual void verifyEvents()
Validates the contents of the set of events.
bool doOnEntry()
Checks if on entry flag is true.
static const int NOP_EVT
Signifies that no event has occurred.
int getCurrState() const
Fetches the model's current state.
void defineState(int value, const std::string &label, StateHandler handler, const StatePausing &state_pausing=STATE_PAUSE_NEVER)
Adds an state value and associated label to the set of states.
void startModel(const int start_state)
Begins execution of the model.
const EventPtr & getEvent(int value)
Fetches the event referred to by value.
virtual void defineStates()
Populates the set of states.
virtual void runModel(int event)
Processes events through the state model.
void transition(int state, int event)
Sets up the model to transition into given state with a given event.
int getNextEvent() const
Fetches the model's next event.
int getPrevState() const
Fetches the model's previous state.
const StatePtr getState(int value)
Fetches the state referred to by value.
void postNextEvent(int event)
Sets the next event to the given event value.
Utility class to measure code execution times.
Definition stopwatch.h:35
void stop()
Stops the stopwatch.
Definition stopwatch.cc:34
std::string logFormatLastDuration() const
Returns the last measured duration in the format directly usable in log messages.
Definition stopwatch.cc:74
This file contains several functions and constants that are used for handling commands and responses ...
if(!(yy_init))
Definition d2_lexer.cc:1517
#define isc_throw(type, stream)
A shortcut macro to insert known values into exception arguments.
An abstract API for lease database.
#define LOG_ERROR(LOGGER, MESSAGE)
Macro to conveniently test error output and log it.
Definition macros.h:32
#define LOG_INFO(LOGGER, MESSAGE)
Macro to conveniently test info output and log it.
Definition macros.h:20
#define LOG_WARN(LOGGER, MESSAGE)
Macro to conveniently test warn output and log it.
Definition macros.h:26
const int CONTROL_RESULT_EMPTY
Status code indicating that the specified command was completed correctly, but failed to produce any ...
const char * CONTROL_TEXT
String used for storing textual description ("text").
ConstElementPtr parseAnswer(int &rcode, const ConstElementPtr &msg)
Parses a standard config/command level answer and returns arguments or text status code.
constexpr long TIMEOUT_DEFAULT_HTTP_CLIENT_REQUEST
Timeout for the HTTP clients awaiting a response to a request.
Definition timeouts.h:34
const int CONTROL_RESULT_ERROR
Status code indicating a general failure.
ConstElementPtr createAnswer()
Creates a standard config/command level success answer message (i.e.
const int CONTROL_RESULT_CONFLICT
Status code indicating that the command was unsuccessful due to a conflict between the command argume...
const int CONTROL_RESULT_COMMAND_UNSUPPORTED
Status code indicating that the specified command is not supported.
const char * CONTROL_RESULT
String used for result, i.e. integer status ("result").
const int CONTROL_RESULT_SUCCESS
Status code indicating a successful operation.
boost::shared_ptr< const Element > ConstElementPtr
Definition data.h:30
boost::shared_ptr< Element > ElementPtr
Definition data.h:29
boost::shared_ptr< isc::dhcp::Pkt > PktPtr
A pointer to either Pkt4 or Pkt6 packet.
Definition pkt.h:1006
std::string ClientClass
Defines a single class name.
Definition classify.h:45
boost::shared_ptr< Lease4Collection > Lease4CollectionPtr
A shared pointer to the collection of IPv4 leases.
Definition lease.h:551
boost::shared_ptr< Pkt4 > Pkt4Ptr
A pointer to Pkt4 object.
Definition pkt4.h:556
boost::shared_ptr< Lease6 > Lease6Ptr
Pointer to a Lease6 structure.
Definition lease.h:556
boost::shared_ptr< Lease > LeasePtr
Pointer to the lease object.
Definition lease.h:27
boost::shared_ptr< NetworkState > NetworkStatePtr
Pointer to the NetworkState object.
boost::shared_ptr< Lease6Collection > Lease6CollectionPtr
A shared pointer to the collection of IPv6 leases.
Definition lease.h:724
boost::shared_ptr< Pkt6 > Pkt6Ptr
A pointer to Pkt6 packet.
Definition pkt6.h:31
std::vector< Lease4Ptr > Lease4Collection
A collection of IPv4 leases.
Definition lease.h:548
boost::shared_ptr< Lease4 > Lease4Ptr
Pointer to a Lease4 structure.
Definition lease.h:343
const isc::log::MessageID HA_INVALID_PARTNER_STATE_LOAD_BALANCING
Definition ha_messages.h:53
const isc::log::MessageID HA_RESUME_CLIENT_LISTENER_FAILED
const isc::log::MessageID HA_LOCAL_DHCP_ENABLE
Definition ha_messages.h:92
const isc::log::MessageID HA_LEASES_BACKLOG_NOTHING_TO_SEND
Definition ha_messages.h:69
const isc::log::MessageID HA_LEASES_BACKLOG_FAILED
Definition ha_messages.h:68
const isc::log::MessageID HA_SYNC_FAILED
const isc::log::MessageID HA_TERMINATED_RESTART_PARTNER
const int HA_PASSIVE_BACKUP_ST
In passive-backup state with a single active server and backup servers.
const int HA_HOT_STANDBY_ST
Hot standby state.
const isc::log::MessageID HA_INVALID_PARTNER_STATE_COMMUNICATION_RECOVERY
Definition ha_messages.h:51
const isc::log::MessageID HA_LEASES_BACKLOG_SUCCESS
Definition ha_messages.h:71
const int HA_COMMUNICATION_RECOVERY_ST
Communication recovery state.
const isc::log::MessageID HA_STATE_MACHINE_CONTINUED
isc::log::Logger ha_logger("ha-hooks")
Definition ha_log.h:17
const isc::log::MessageID HA_LEASES_SYNC_FAILED
Definition ha_messages.h:74
const isc::log::MessageID HA_SYNC_SUCCESSFUL
const int HA_UNAVAILABLE_ST
Special state indicating that this server is unable to communicate with the partner.
const isc::log::MessageID HA_CONFIG_LEASE_UPDATES_DISABLED_REMINDER
Definition ha_messages.h:34
const isc::log::MessageID HA_SERVICE_STARTED
const int HA_TERMINATED_ST
HA service terminated state.
const int HA_IN_MAINTENANCE_ST
In maintenance state.
const int HA_LOAD_BALANCING_ST
Load balancing state.
const isc::log::MessageID HA_DHCP_ENABLE_FAILED
Definition ha_messages.h:44
const int HA_DOWN_ST
Server down state, i.e. lost connection to lease storage database.
const isc::log::MessageID HA_LEASE_UPDATE_DELETE_FAILED_ON_PEER
Definition ha_messages.h:84
const isc::log::MessageID HA_LEASES_BACKLOG_START
Definition ha_messages.h:70
const isc::log::MessageID HA_SYNC_START
const isc::log::MessageID HA_HEARTBEAT_FAILED
Definition ha_messages.h:46
const int HA_PARTNER_DOWN_ST
Partner down state.
const isc::log::MessageID HA_LEASE_UPDATES_ENABLED
Definition ha_messages.h:80
const isc::log::MessageID HA_INVALID_PARTNER_STATE_HOT_STANDBY
Definition ha_messages.h:52
const isc::log::MessageID HA_STATE_MACHINE_PAUSED
const isc::log::MessageID HA_TERMINATED
const isc::log::MessageID HA_DHCP_DISABLE_FAILED
Definition ha_messages.h:42
boost::shared_ptr< HAConfig > HAConfigPtr
Pointer to the High Availability configuration structure.
Definition ha_config.h:39
const isc::log::MessageID HA_MAINTENANCE_STARTED_IN_PARTNER_DOWN
const int HA_PARTNER_IN_MAINTENANCE_ST
Partner in-maintenance state.
const isc::log::MessageID HA_MAINTENANCE_NOTIFY_FAILED
Definition ha_messages.h:97
const int HA_WAITING_ST
Server waiting state, i.e. waiting for another server to be ready.
HAServerType
Lists possible server types for which HA service is created.
const int HA_BACKUP_ST
Backup state.
const isc::log::MessageID HA_PAUSE_CLIENT_LISTENER_ILLEGAL
const isc::log::MessageID HA_PAUSE_CLIENT_LISTENER_FAILED
const isc::log::MessageID HA_MAINTENANCE_SHUTDOWN_SAFE
Definition ha_messages.h:99
const isc::log::MessageID HA_MAINTENANCE_NOTIFY_CANCEL_FAILED
Definition ha_messages.h:95
const isc::log::MessageID HA_LEASE_UPDATE_CONFLICT
Definition ha_messages.h:82
const isc::log::MessageID HA_LEASE_UPDATES_DISABLED
Definition ha_messages.h:79
const isc::log::MessageID HA_LOCAL_DHCP_DISABLE
Definition ha_messages.h:91
const int HA_SYNCING_ST
Synchronizing database state.
const isc::log::MessageID HA_RESET_FAILED
const isc::log::MessageID HA_STATE_TRANSITION
const isc::log::MessageID HA_CONFIG_LEASE_SYNCING_DISABLED_REMINDER
Definition ha_messages.h:31
std::string stateToString(int state)
Returns state name.
const int HA_READY_ST
Server ready state, i.e. synchronized database, can enable DHCP service.
const isc::log::MessageID HA_TERMINATED_PARTNER_DID_NOT_RESTART
const isc::log::MessageID HA_SYNC_COMPLETE_NOTIFY_FAILED
const isc::log::MessageID HA_MAINTENANCE_STARTED
const isc::log::MessageID HA_LEASE_UPDATE_CREATE_UPDATE_FAILED_ON_PEER
Definition ha_messages.h:83
const isc::log::MessageID HA_LEASE_UPDATE_FAILED
Definition ha_messages.h:85
const isc::log::MessageID HA_STATE_TRANSITION_PASSIVE_BACKUP
boost::shared_ptr< ParkingLotHandle > ParkingLotHandlePtr
Pointer to the parking lot handle.
boost::shared_ptr< PostHttpRequestJson > PostHttpRequestJsonPtr
Pointer to PostHttpRequestJson.
boost::shared_ptr< HttpAuthConfig > HttpAuthConfigPtr
Type of shared pointers to HTTP authentication configuration.
Definition auth_config.h:97
boost::shared_ptr< HttpResponseJson > HttpResponseJsonPtr
Pointer to the HttpResponseJson object.
boost::shared_ptr< HttpResponse > HttpResponsePtr
Pointer to the HttpResponse object.
Definition response.h:82
const char * MessageID
std::string ptimeToText(boost::posix_time::ptime t, size_t fsecs_precision=MAX_FSECS_PRECISION)
Converts ptime structure to text.
Defines the logger used by the top-level component of kea-lfc.
int http_response(CalloutHandle &handle)
This callout is called at the "http_response" hook.
static constexpr uint32_t STATE_RELEASED
Released lease held in the database for lease affinity.
Definition lease.h:80
HTTP request/response timeout value.
static const HttpVersion & HTTP_11()
HTTP version 1.1.
Definition http_types.h:59