Kea  2.1.7-git
ha_service.cc
Go to the documentation of this file.
1 // Copyright (C) 2018-2022 Internet Systems Consortium, Inc. ("ISC")
2 //
3 // This Source Code Form is subject to the terms of the Mozilla Public
4 // License, v. 2.0. If a copy of the MPL was not distributed with this
5 // file, You can obtain one at http://mozilla.org/MPL/2.0/.
6 
7 #include <config.h>
8 
9 #include <command_creator.h>
10 #include <ha_log.h>
11 #include <ha_service.h>
12 #include <ha_service_states.h>
13 #include <cc/command_interpreter.h>
14 #include <cc/data.h>
16 #include <config/timeouts.h>
17 #include <dhcp/iface_mgr.h>
18 #include <dhcpsrv/cfgmgr.h>
19 #include <dhcpsrv/lease_mgr.h>
21 #include <exceptions/exceptions.h>
22 #include <http/date_time.h>
23 #include <http/response_json.h>
24 #include <http/post_request_json.h>
26 #include <util/stopwatch.h>
27 #include <boost/pointer_cast.hpp>
28 #include <boost/make_shared.hpp>
29 #include <boost/weak_ptr.hpp>
30 #include <functional>
31 #include <sstream>
32 
33 using namespace isc::asiolink;
34 using namespace isc::config;
35 using namespace isc::data;
36 using namespace isc::dhcp;
37 using namespace isc::hooks;
38 using namespace isc::http;
39 using namespace isc::log;
40 using namespace isc::util;
41 namespace ph = std::placeholders;
42 
43 namespace {
44 
46 class CommandUnsupportedError : public CtrlChannelError {
47 public:
48  CommandUnsupportedError(const char* file, size_t line, const char* what) :
49  CtrlChannelError(file, line, what) {}
50 };
51 
52 }
53 
54 namespace isc {
55 namespace ha {
56 
57 const int HAService::HA_HEARTBEAT_COMPLETE_EVT;
58 const int HAService::HA_LEASE_UPDATES_COMPLETE_EVT;
59 const int HAService::HA_SYNCING_FAILED_EVT;
60 const int HAService::HA_SYNCING_SUCCEEDED_EVT;
61 const int HAService::HA_MAINTENANCE_NOTIFY_EVT;
62 const int HAService::HA_MAINTENANCE_START_EVT;
63 const int HAService::HA_MAINTENANCE_CANCEL_EVT;
64 const int HAService::HA_CONTROL_RESULT_MAINTENANCE_NOT_ALLOWED;
65 const int HAService::HA_SYNCED_PARTNER_UNAVAILABLE_EVT;
66 
67 HAService::HAService(const IOServicePtr& io_service, const NetworkStatePtr& network_state,
68  const HAConfigPtr& config, const HAServerType& server_type)
69  : io_service_(io_service), network_state_(network_state), config_(config),
70  server_type_(server_type), client_(), listener_(), communication_state_(),
71  query_filter_(config), mutex_(), pending_requests_(),
72  lease_update_backlog_(config->getDelayedUpdatesLimit()),
73  sync_complete_notified_(false) {
74 
75  if (server_type == HAServerType::DHCPv4) {
77 
78  } else {
80  }
81 
82  network_state_->reset(NetworkState::Origin::HA_COMMAND);
83 
85 
86  // Create the client and(or) listener as appropriate.
87  if (!config_->getEnableMultiThreading()) {
88  // Not configured for multi-threading, start a client in ST mode.
89  client_.reset(new HttpClient(*io_service_, 0));
90  } else {
91  // Create an MT-mode client.
92  client_.reset(new HttpClient(*io_service_,
93  config_->getHttpClientThreads(), true));
94 
95  // If we're configured to use our own listener create and start it.
96  if (config_->getHttpDedicatedListener()) {
97  // Get the server address and port from this server's URL.
98  auto my_url = config_->getThisServerConfig()->getUrl();
99  IOAddress server_address(IOAddress::IPV4_ZERO_ADDRESS());
100  try {
101  // Since we do not currently support hostname resolution,
102  // we need to make sure we have an IP address here.
103  server_address = IOAddress(my_url.getStrippedHostname());
104  } catch (const std::exception& ex) {
105  isc_throw(Unexpected, "server Url:" << my_url.getStrippedHostname()
106  << " is not a valid IP address");
107  }
108 
109  // Fetch how many threads the listener will use.
110  uint32_t listener_threads = config_->getHttpListenerThreads();
111 
112  // Fetch the TLS context.
113  auto tls_context = config_->getThisServerConfig()->getTlsContext();
114 
115  // Instantiate the listener.
116  listener_.reset(new CmdHttpListener(server_address, my_url.getPort(),
117  listener_threads, tls_context));
118  // Set the command filter when enabled.
119  if (config_->getRestrictCommands()) {
120  if (server_type == HAServerType::DHCPv4) {
121  CmdResponseCreator::command_accept_list_ =
123  } else {
124  CmdResponseCreator::command_accept_list_ =
126  }
127  }
128  }
129  }
130 
132  .arg(HAConfig::HAModeToString(config->getHAMode()))
133  .arg(HAConfig::PeerConfig::roleToString(config->getThisServerConfig()->getRole()));
134 }
135 
137  // Stop client and/or listener.
139 
140  network_state_->reset(NetworkState::Origin::HA_COMMAND);
141 }
142 
143 void
145  StateModel::defineEvents();
146 
147  defineEvent(HA_HEARTBEAT_COMPLETE_EVT, "HA_HEARTBEAT_COMPLETE_EVT");
148  defineEvent(HA_LEASE_UPDATES_COMPLETE_EVT, "HA_LEASE_UPDATES_COMPLETE_EVT");
149  defineEvent(HA_SYNCING_FAILED_EVT, "HA_SYNCING_FAILED_EVT");
150  defineEvent(HA_SYNCING_SUCCEEDED_EVT, "HA_SYNCING_SUCCEEDED_EVT");
151  defineEvent(HA_MAINTENANCE_NOTIFY_EVT, "HA_MAINTENANCE_NOTIFY_EVT");
152  defineEvent(HA_MAINTENANCE_START_EVT, "HA_MAINTENANCE_START_EVT");
153  defineEvent(HA_MAINTENANCE_CANCEL_EVT, "HA_MAINTENANCE_CANCEL_EVT");
154  defineEvent(HA_SYNCED_PARTNER_UNAVAILABLE_EVT, "HA_SYNCED_PARTNER_UNAVAILABLE_EVT");
155 }
156 
157 void
159  StateModel::verifyEvents();
160 
169 }
170 
171 void
173  StateModel::defineStates();
174 
176  std::bind(&HAService::backupStateHandler, this),
177  config_->getStateMachineConfig()->getStateConfig(HA_BACKUP_ST)->getPausing());
178 
180  std::bind(&HAService::communicationRecoveryHandler, this),
181  config_->getStateMachineConfig()->getStateConfig(HA_COMMUNICATION_RECOVERY_ST)->getPausing());
182 
184  std::bind(&HAService::normalStateHandler, this),
185  config_->getStateMachineConfig()->getStateConfig(HA_HOT_STANDBY_ST)->getPausing());
186 
188  std::bind(&HAService::normalStateHandler, this),
189  config_->getStateMachineConfig()->getStateConfig(HA_LOAD_BALANCING_ST)->getPausing());
190 
192  std::bind(&HAService::inMaintenanceStateHandler, this),
193  config_->getStateMachineConfig()->getStateConfig(HA_IN_MAINTENANCE_ST)->getPausing());
194 
196  std::bind(&HAService::partnerDownStateHandler, this),
197  config_->getStateMachineConfig()->getStateConfig(HA_PARTNER_DOWN_ST)->getPausing());
198 
201  config_->getStateMachineConfig()->getStateConfig(HA_PARTNER_IN_MAINTENANCE_ST)->getPausing());
202 
204  std::bind(&HAService::passiveBackupStateHandler, this),
205  config_->getStateMachineConfig()->getStateConfig(HA_PASSIVE_BACKUP_ST)->getPausing());
206 
208  std::bind(&HAService::readyStateHandler, this),
209  config_->getStateMachineConfig()->getStateConfig(HA_READY_ST)->getPausing());
210 
212  std::bind(&HAService::syncingStateHandler, this),
213  config_->getStateMachineConfig()->getStateConfig(HA_SYNCING_ST)->getPausing());
214 
216  std::bind(&HAService::terminatedStateHandler, this),
217  config_->getStateMachineConfig()->getStateConfig(HA_TERMINATED_ST)->getPausing());
218 
220  std::bind(&HAService::waitingStateHandler, this),
221  config_->getStateMachineConfig()->getStateConfig(HA_WAITING_ST)->getPausing());
222 }
223 
224 void
226  if (doOnEntry()) {
229 
230  // Log if the state machine is paused.
232  }
233 
234  // There is nothing to do in that state. This server simply receives
235  // lease updates from the partners.
237 }
238 
239 void
241  if (doOnEntry()) {
244 
245  // Log if the state machine is paused.
247  }
248 
250 
253 
254  // Check if the clock skew is still acceptable. If not, transition to
255  // the terminated state.
256  } else if (shouldTerminate()) {
258 
259  } else if (isPartnerStateInvalid()) {
261 
262  } else {
263 
264  // Transitions based on the partner's state.
265  switch (communication_state_->getPartnerState()) {
268  break;
269 
270  case HA_PARTNER_DOWN_ST:
272  break;
273 
276  break;
277 
278  case HA_TERMINATED_ST:
280  break;
281 
282  case HA_UNAVAILABLE_ST:
283  if (shouldPartnerDown()) {
285 
286  } else {
288  }
289  break;
290 
291  case HA_WAITING_ST:
292  case HA_SYNCING_ST:
293  case HA_READY_ST:
294  // The partner seems to be waking up, perhaps after communication-recovery.
295  // If our backlog queue is overflown we need to synchronize our lease database.
296  // There is no need to send ha-reset to the partner because the partner is
297  // already synchronizing its lease database.
298  if (!communication_state_->isCommunicationInterrupted() &&
301  } else {
302  // Backlog was not overflown, so there is no need to synchronize our
303  // lease database. Let's wait until our partner completes synchronization
304  // and transitions to the load-balancing state.
306  }
307  break;
308 
309  default:
310  // If the communication is still interrupted, let's continue sitting
311  // in this state until it is resumed or until the transition to the
312  // partner-down state, depending on what happens first.
313  if (communication_state_->isCommunicationInterrupted()) {
315  break;
316  }
317 
318  // The communication has been resumed. The partner server must be in a state
319  // in which it can receive outstanding lease updates we collected. The number of
320  // outstanding lease updates must not exceed the configured limit. Finally, the
321  // lease updates must be successfully sent. If that all works, we will transition
322  // to the normal operation.
323  if ((communication_state_->getPartnerState() == getNormalState()) ||
324  (communication_state_->getPartnerState() == HA_COMMUNICATION_RECOVERY_ST)) {
326  // If our lease backlog was overflown or we were unable to send lease
327  // updates to the partner we should notify the partner that it should
328  // synchronize the lease database. We do it by sending ha-reset command.
329  if (sendHAReset()) {
331  }
332  break;
333  }
334  // The backlog was not overflown and we successfully sent our lease updates.
335  // We can now transition to the normal operation state. If the partner
336  // fails to send his outstanding lease updates to us it should send the
337  // ha-reset command to us.
339  break;
340  }
341 
342  // The partner appears to be in unexpected state, we have exceeded the number
343  // of lease updates in a backlog or an attempt to send lease updates failed.
344  // In all these cases we follow plan B and transition to the waiting state.
345  // The server will then attempt to synchronize the entire lease database.
347  }
348  }
349 
350  // When exiting this state we must ensure that lease updates backlog is cleared.
351  if (doOnExit()) {
353  }
354 }
355 
356 void
358  // If we are transitioning from another state, we have to define new
359  // serving scopes appropriate for the new state. We don't do it if
360  // we remain in this state.
361  if (doOnEntry()) {
364 
365  // Log if the state machine is paused.
367  }
368 
370 
373  return;
374  }
375 
376  // Check if the clock skew is still acceptable. If not, transition to
377  // the terminated state.
378  if (shouldTerminate()) {
380  return;
381  }
382 
383  // Check if the partner state is valid per current configuration. If it is
384  // in an invalid state let's transition to the waiting state and stay there
385  // until the configuration is corrected.
386  if (isPartnerStateInvalid()) {
388  return;
389  }
390 
391  switch (communication_state_->getPartnerState()) {
394  break;
395 
396  case HA_PARTNER_DOWN_ST:
398  break;
399 
402  break;
403 
404  case HA_TERMINATED_ST:
406  break;
407 
408  case HA_UNAVAILABLE_ST:
409  if (shouldPartnerDown()) {
411 
412  } else if (config_->amAllowingCommRecovery()) {
414 
415  } else {
417  }
418  break;
419 
420  default:
422  }
423 
424  if (doOnExit()) {
425  // Do nothing here but doOnExit() call clears the "on exit" flag
426  // when transitioning to the communication-recovery state. In that
427  // state we need this flag to be cleared.
428  }
429 }
430 
431 void
433  // If we are transitioning from another state, we have to define new
434  // serving scopes appropriate for the new state. We don't do it if
435  // we remain in this state.
436  if (doOnEntry()) {
437  // In this state the server remains silent and waits for being
438  // shutdown.
441 
442  // Log if the state machine is paused.
444 
446  }
447 
449 
450  // We don't transition out of this state unless explicitly mandated
451  // by the administrator via a dedicated command which cancels
452  // the maintenance.
454 }
455 
456 void
458  // If we are transitioning from another state, we have to define new
459  // serving scopes appropriate for the new state. We don't do it if
460  // we remain in this state.
461  if (doOnEntry()) {
462 
463  bool maintenance = (getLastEvent() == HA_MAINTENANCE_START_EVT);
464 
465  // It may be administratively disabled to handle partner's scope
466  // in case of failure. If this is the case we'll just handle our
467  // default scope (or no scope at all). The user will need to
468  // manually enable this server to handle partner's scope.
469  // If we're in the maintenance mode we serve all scopes because
470  // it is not a failover situation.
471  if (maintenance || config_->getThisServerConfig()->isAutoFailover()) {
473  } else {
475  }
477 
478  // Log if the state machine is paused.
480 
481  if (maintenance) {
482  // If we ended up in the partner-down state as a result of
483  // receiving the ha-maintenance-start command let's log it.
485  }
486 
488  // Partner sent the ha-sync-complete-notify command to indicate that
489  // it has successfully synchronized its lease database but this server
490  // was unable to send heartbeat to this server. Enable the DHCP service
491  // and continue serving the clients in the partner-down state until the
492  // communication with the partner is fixed.
494  }
495 
497 
500  return;
501  }
502 
503  // Check if the clock skew is still acceptable. If not, transition to
504  // the terminated state.
505  if (shouldTerminate()) {
507  return;
508  }
509 
510  // Check if the partner state is valid per current configuration. If it is
511  // in an invalid state let's transition to the waiting state and stay there
512  // until the configuration is corrected.
513  if (isPartnerStateInvalid()) {
515  return;
516  }
517 
518  switch (communication_state_->getPartnerState()) {
520  case HA_PARTNER_DOWN_ST:
523  break;
524 
525  case HA_READY_ST:
526  // If partner allocated new leases for which it didn't send lease updates
527  // to us we should synchronize our database.
528  if (communication_state_->hasPartnerNewUnsentUpdates()) {
530  } else {
531  // We did not miss any lease updates. There is no need to synchronize
532  // the database.
534  }
535  break;
536 
537  case HA_TERMINATED_ST:
539  break;
540 
541  default:
543  }
544 }
545 
546 void
548  // If we are transitioning from another state, we have to define new
549  // serving scopes appropriate for the new state. We don't do it if
550  // we remain in this state.
551  if (doOnEntry()) {
553 
555 
556  // Log if the state machine is paused.
558 
560  }
561 
563 
564  if (isModelPaused()) {
566  return;
567  }
568 
569  // Check if the clock skew is still acceptable. If not, transition to
570  // the terminated state.
571  if (shouldTerminate()) {
573  return;
574  }
575 
576  switch (communication_state_->getPartnerState()) {
577  case HA_UNAVAILABLE_ST:
579  break;
580  default:
582  }
583 }
584 
585 void
587  // If we are transitioning from another state, we have to define new
588  // serving scopes appropriate for the new state. We don't do it if
589  // we remain in this state.
590  if (doOnEntry()) {
593 
594  // In the passive-backup state we don't send heartbeat.
595  communication_state_->stopHeartbeat();
596 
597  // Log if the state machine is paused.
599  }
601 }
602 
603 void
605  // If we are transitioning from another state, we have to define new
606  // serving scopes appropriate for the new state. We don't do it if
607  // we remain in this state.
608  if (doOnEntry()) {
611 
612  // Log if the state machine is paused.
614  }
615 
617 
620  return;
621  }
622 
623  // Check if the clock skew is still acceptable. If not, transition to
624  // the terminated state.
625  if (shouldTerminate()) {
627  return;
628  }
629 
630  // Check if the partner state is valid per current configuration. If it is
631  // in an invalid state let's transition to the waiting state and stay there
632  // until the configuration is corrected.
633  if (isPartnerStateInvalid()) {
635  return;
636  }
637 
638  switch (communication_state_->getPartnerState()) {
639  case HA_HOT_STANDBY_ST:
643  break;
644 
647  break;
648 
651  break;
652 
653  case HA_READY_ST:
654  // If both servers are ready, the primary server "wins" and is
655  // transitioned first.
656  if (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::PRIMARY) {
659  } else {
661  }
662  break;
663 
664  case HA_TERMINATED_ST:
666  break;
667 
668  case HA_UNAVAILABLE_ST:
669  if (shouldPartnerDown()) {
671 
672  } else {
674  }
675  break;
676 
677  default:
679  }
680 }
681 
682 void
684  // If we are transitioning from another state, we have to define new
685  // serving scopes appropriate for the new state. We don't do it if
686  // we remain in this state.
687  if (doOnEntry()) {
690 
691  // Log if the state machine is paused.
693  }
694 
697  return;
698  }
699 
700  // Check if the clock skew is still acceptable. If not, transition to
701  // the terminated state.
702  if (shouldTerminate()) {
704  return;
705  }
706 
707  // Check if the partner state is valid per current configuration. If it is
708  // in an invalid state let's transition to the waiting state and stay there
709  // until the configuration is corrected.
710  if (isPartnerStateInvalid()) {
712  return;
713  }
714 
715  // We don't want to perform synchronous attempt to synchronize with
716  // a partner until we know that the partner is responding. Therefore,
717  // we wait for the heartbeat to complete successfully before we
718  // initiate the synchronization.
719  switch (communication_state_->getPartnerState()) {
720  case HA_TERMINATED_ST:
722  return;
723 
724  case HA_UNAVAILABLE_ST:
725  // If the partner appears to be offline, let's transition to the partner
726  // down state. Otherwise, we'd be stuck trying to synchronize with a
727  // dead partner.
728  if (shouldPartnerDown()) {
730 
731  } else {
733  }
734  break;
735 
736  default:
737  // We don't want the heartbeat to interfere with the synchronization,
738  // so let's temporarily stop it.
739  communication_state_->stopHeartbeat();
740 
741  // Timeout is configured in milliseconds. Need to convert to seconds.
742  unsigned int dhcp_disable_timeout =
743  static_cast<unsigned int>(config_->getSyncTimeout() / 1000);
744  if (dhcp_disable_timeout == 0) {
745  ++dhcp_disable_timeout;
746  }
747 
748  // Perform synchronous leases update.
749  std::string status_message;
750  int sync_status = synchronize(status_message,
751  config_->getFailoverPeerConfig()->getName(),
752  dhcp_disable_timeout);
753 
754  // If the leases synchronization was successful, let's transition
755  // to the ready state.
756  if (sync_status == CONTROL_RESULT_SUCCESS) {
758 
759  } else {
760  // If the synchronization was unsuccessful we're back to the
761  // situation that the partner is unavailable and therefore
762  // we stay in the syncing state.
764  }
765  }
766 
767  // Make sure that the heartbeat is re-enabled.
769 }
770 
771 void
773  // If we are transitioning from another state, we have to define new
774  // serving scopes appropriate for the new state. We don't do it if
775  // we remain in this state.
776  if (doOnEntry()) {
779 
780  // In the terminated state we don't send heartbeat.
781  communication_state_->stopHeartbeat();
782 
783  // Log if the state machine is paused.
785 
787  }
788 
790 }
791 
792 void
794  // If we are transitioning from another state, we have to define new
795  // serving scopes appropriate for the new state. We don't do it if
796  // we remain in this state.
797  if (doOnEntry()) {
800 
801  // Log if the state machine is paused.
803  }
804 
805  // Only schedule the heartbeat for non-backup servers.
806  if ((config_->getHAMode() != HAConfig::PASSIVE_BACKUP) &&
807  (config_->getThisServerConfig()->getRole() != HAConfig::PeerConfig::BACKUP)) {
809  }
810 
813  return;
814  }
815 
816  // Backup server must remain in its own state.
817  if (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::BACKUP) {
819  return;
820  }
821 
822  // We're not a backup server, so we're either primary or secondary. If this is
823  // a passive-backup mode of operation, we're primary and we should transition
824  // to the passive-backup state.
825  if (config_->getHAMode() == HAConfig::PASSIVE_BACKUP) {
827  return;
828  }
829 
830  // Check if the clock skew is still acceptable. If not, transition to
831  // the terminated state.
832  if (shouldTerminate()) {
834  return;
835  }
836 
837  // Check if the partner state is valid per current configuration. If it is
838  // in an invalid state let's sit in the waiting state until the configuration
839  // is corrected.
840  if (isPartnerStateInvalid()) {
842  return;
843  }
844 
845  switch (communication_state_->getPartnerState()) {
847  case HA_HOT_STANDBY_ST:
850  case HA_PARTNER_DOWN_ST:
852  case HA_READY_ST:
853  // If we're configured to not synchronize lease database, proceed directly
854  // to the "ready" state.
855  verboseTransition(config_->amSyncingLeases() ? HA_SYNCING_ST : HA_READY_ST);
856  break;
857 
858  case HA_SYNCING_ST:
860  break;
861 
862  case HA_TERMINATED_ST:
863  // We have checked above whether the clock skew is exceeding the threshold
864  // and we should terminate. If we're here, it means that the clock skew
865  // is acceptable. The partner may be still in the terminated state because
866  // it hasn't been restarted yet. Probably, this server is the first one
867  // being restarted after syncing the clocks. Let's just sit in the waiting
868  // state until the partner gets restarted.
871  break;
872 
873  case HA_WAITING_ST:
874  // If both servers are waiting, the primary server 'wins' and is
875  // transitioned to the next state first.
876  if (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::PRIMARY) {
877  // If we're configured to not synchronize lease database, proceed directly
878  // to the "ready" state.
879  verboseTransition(config_->amSyncingLeases() ? HA_SYNCING_ST : HA_READY_ST);
880 
881  } else {
883  }
884  break;
885 
886  case HA_UNAVAILABLE_ST:
887  if (shouldPartnerDown()) {
889 
890  } else {
892  }
893  break;
894 
895  default:
897  }
898 }
899 
900 void
901 HAService::verboseTransition(const unsigned state) {
902  // Get current and new state name.
903  std::string current_state_name = getStateLabel(getCurrState());
904  std::string new_state_name = getStateLabel(state);
905 
906  // Turn them to upper case so as they are better visible in the logs.
907  boost::to_upper(current_state_name);
908  boost::to_upper(new_state_name);
909 
910  if (config_->getHAMode() != HAConfig::PASSIVE_BACKUP) {
911  // If this is load-balancing or hot-standby mode we also want to log
912  // partner's state.
913  auto partner_state = communication_state_->getPartnerState();
914  std::string partner_state_name = getStateLabel(partner_state);
915  boost::to_upper(partner_state_name);
916 
917  // Log the transition.
919  .arg(current_state_name)
920  .arg(new_state_name)
921  .arg(partner_state_name);
922 
923  } else {
924  // In the passive-backup mode we don't know the partner's state.
926  .arg(current_state_name)
927  .arg(new_state_name);
928  }
929 
930  // If we're transitioning directly from the "waiting" to "ready"
931  // state it indicates that the database synchronization is
932  // administratively disabled. Let's remind the user about this
933  // configuration setting.
934  if ((state == HA_READY_ST) && (getCurrState() == HA_WAITING_ST)) {
936  }
937 
938  // Do the actual transition.
939  transition(state, getNextEvent());
940 
941  // Inform the administrator whether or not lease updates are generated.
942  // Updates are never generated by a backup server so it doesn't make
943  // sense to log anything for the backup server.
944  if ((config_->getHAMode() != HAConfig::PASSIVE_BACKUP) &&
945  (config_->getThisServerConfig()->getRole() != HAConfig::PeerConfig::BACKUP)) {
946  if (shouldSendLeaseUpdates(config_->getFailoverPeerConfig())) {
948  .arg(new_state_name);
949 
950  } else if (!config_->amSendingLeaseUpdates()) {
951  // Lease updates are administratively disabled.
953  .arg(new_state_name);
954 
955  } else {
956  // Lease updates are not administratively disabled, but they
957  // are not issued because this is the backup server or because
958  // in this state the server should not generate lease updates.
960  .arg(new_state_name);
961  }
962  }
963 }
964 
965 int
967  if (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::BACKUP) {
968  return (HA_BACKUP_ST);
969  }
970 
971  switch (config_->getHAMode()) {
973  return (HA_LOAD_BALANCING_ST);
975  return (HA_HOT_STANDBY_ST);
976  default:
977  return (HA_PASSIVE_BACKUP_ST);
978  }
979 }
980 
981 bool
983  if (isModelPaused()) {
985  unpauseModel();
986  return (true);
987  }
988  return (false);
989 }
990 
991 void
993  // Inform the administrator if the state machine is paused.
994  if (isModelPaused()) {
995  std::string state_name = stateToString(getCurrState());
996  boost::to_upper(state_name);
998  .arg(state_name);
999  }
1000 }
1001 
1002 void
1005 }
1006 
1007 bool
1009  return (inScopeInternal(query4));
1010 }
1011 
1012 bool
1014  return (inScopeInternal(query6));
1015 }
1016 
1017 template<typename QueryPtrType>
1018 bool
1019 HAService::inScopeInternal(QueryPtrType& query) {
1020  // Check if the query is in scope (should be processed by this server).
1021  std::string scope_class;
1022  const bool in_scope = query_filter_.inScope(query, scope_class);
1023  // Whether or not the query is going to be processed by this server,
1024  // we associate the query with the appropriate class.
1025  query->addClass(dhcp::ClientClass(scope_class));
1026  // The following is the part of the server failure detection algorithm.
1027  // If the query should be processed by the partner we need to check if
1028  // the partner responds. If the number of unanswered queries exceeds a
1029  // configured threshold, we will consider the partner to be offline.
1030  if (!in_scope && communication_state_->isCommunicationInterrupted()) {
1031  communication_state_->analyzeMessage(query);
1032  }
1033  // Indicate if the query is in scope.
1034  return (in_scope);
1035 }
1036 
1037 void
1039  std::string current_state_name = getStateLabel(getCurrState());
1040  boost::to_upper(current_state_name);
1041 
1042  // DHCP service should be enabled in the following states.
1043  const bool should_enable = ((getCurrState() == HA_COMMUNICATION_RECOVERY_ST) ||
1050 
1051  if (!should_enable && network_state_->isServiceEnabled()) {
1052  std::string current_state_name = getStateLabel(getCurrState());
1053  boost::to_upper(current_state_name);
1055  .arg(config_->getThisServerName())
1056  .arg(current_state_name);
1057  network_state_->disableService(NetworkState::Origin::HA_COMMAND);
1058 
1059  } else if (should_enable && !network_state_->isServiceEnabled()) {
1060  std::string current_state_name = getStateLabel(getCurrState());
1061  boost::to_upper(current_state_name);
1063  .arg(config_->getThisServerName())
1064  .arg(current_state_name);
1065  network_state_->enableService(NetworkState::Origin::HA_COMMAND);
1066  }
1067 }
1068 
1069 bool
1071  // Checking whether the communication with the partner is OK is the
1072  // first step towards verifying if the server is up.
1073  if (communication_state_->isCommunicationInterrupted()) {
1074  // If the communication is interrupted, we also have to check
1075  // whether the partner answers DHCP requests. The only cases
1076  // when we don't (can't) do it are: the hot standby configuration
1077  // in which this server is a primary and when the DHCP service is
1078  // disabled so we can't analyze incoming traffic. Note that the
1079  // primary server can't check delayed responses to the partner
1080  // because the partner doesn't respond to any queries in this
1081  // configuration.
1082  if (network_state_->isServiceEnabled() &&
1083  ((config_->getHAMode() == HAConfig::LOAD_BALANCING) ||
1084  (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::STANDBY))) {
1085  return (communication_state_->failureDetected());
1086  }
1087 
1088  // Hot standby / primary case.
1089  return (true);
1090  }
1091 
1092  // Shouldn't transition to the partner down state.
1093  return (false);
1094 }
1095 
1096 bool
1098  // Check if skew is fatally large.
1099  bool should_terminate = communication_state_->clockSkewShouldTerminate();
1100 
1101  // If not issue a warning if it's getting large.
1102  if (!should_terminate) {
1103  communication_state_->clockSkewShouldWarn();
1104  }
1105 
1106  return (should_terminate);
1107 }
1108 
1109 bool
1112 }
1113 
1114 bool
1116  switch (communication_state_->getPartnerState()) {
1118  if (config_->getHAMode() != HAConfig::LOAD_BALANCING) {
1120  return (true);
1121  }
1122  break;
1123 
1124  case HA_HOT_STANDBY_ST:
1125  if (config_->getHAMode() != HAConfig::HOT_STANDBY) {
1127  return (true);
1128  }
1129  break;
1130 
1131  case HA_LOAD_BALANCING_ST:
1132  if (config_->getHAMode() != HAConfig::LOAD_BALANCING) {
1134  return (true);
1135  }
1136  break;
1137 
1138  default:
1139  ;
1140  }
1141  return (false);
1142 }
1143 
1144 size_t
1146  const dhcp::Lease4CollectionPtr& leases,
1147  const dhcp::Lease4CollectionPtr& deleted_leases,
1148  const hooks::ParkingLotHandlePtr& parking_lot) {
1149 
1150  // Get configurations of the peers. Exclude this instance.
1151  HAConfig::PeerConfigMap peers_configs = config_->getOtherServersConfig();
1152 
1153  size_t sent_num = 0;
1154 
1155  // Schedule sending lease updates to each peer.
1156  for (auto p = peers_configs.begin(); p != peers_configs.end(); ++p) {
1157  HAConfig::PeerConfigPtr conf = p->second;
1158 
1159  // Check if the lease updates should be queued. This is the case when the
1160  // server is in the communication-recovery state. Queued lease updates may
1161  // be sent when the communication is re-established.
1162  if (shouldQueueLeaseUpdates(conf)) {
1163  // Lease updates for deleted leases.
1164  for (auto l = deleted_leases->begin(); l != deleted_leases->end(); ++l) {
1166  }
1167 
1168  // Lease updates for new allocations and updated leases.
1169  for (auto l = leases->begin(); l != leases->end(); ++l) {
1171  }
1172 
1173  continue;
1174  }
1175 
1176  // Check if the lease update should be sent to the server. If we're in
1177  // the partner-down state we don't send lease updates to the partner.
1178  if (!shouldSendLeaseUpdates(conf)) {
1179  // If we decide to not send the lease updates to an active partner, we
1180  // should make a record of it in the communication state. The partner
1181  // can check if there were any unsent lease updates when he determines
1182  // whether it should synchronize its database or not when it recovers
1183  // from the partner-down state.
1184  if (conf->getRole() != HAConfig::PeerConfig::BACKUP) {
1185  communication_state_->increaseUnsentUpdateCount();
1186  }
1187  continue;
1188  }
1189 
1190  // Lease updates for deleted leases.
1191  for (auto l = deleted_leases->begin(); l != deleted_leases->end(); ++l) {
1193  parking_lot);
1194  }
1195 
1196  // Lease updates for new allocations and updated leases.
1197  for (auto l = leases->begin(); l != leases->end(); ++l) {
1199  parking_lot);
1200  }
1201 
1202  // If we're contacting a backup server from which we don't expect a
1203  // response prior to responding to the DHCP client we don't count
1204  // it.
1205  if ((config_->amWaitingBackupAck() || (conf->getRole() != HAConfig::PeerConfig::BACKUP))) {
1206  ++sent_num;
1207  }
1208  }
1209 
1210  return (sent_num);
1211 }
1212 
1213 size_t
1215  const dhcp::Lease6CollectionPtr& leases,
1216  const dhcp::Lease6CollectionPtr& deleted_leases,
1217  const hooks::ParkingLotHandlePtr& parking_lot) {
1218 
1219  // Get configurations of the peers. Exclude this instance.
1220  HAConfig::PeerConfigMap peers_configs = config_->getOtherServersConfig();
1221 
1222  size_t sent_num = 0;
1223 
1224  // Schedule sending lease updates to each peer.
1225  for (auto p = peers_configs.begin(); p != peers_configs.end(); ++p) {
1226  HAConfig::PeerConfigPtr conf = p->second;
1227 
1228  // Check if the lease updates should be queued. This is the case when the
1229  // server is in the communication-recovery state. Queued lease updates may
1230  // be sent when the communication is re-established.
1231  if (shouldQueueLeaseUpdates(conf)) {
1232  for (auto l = deleted_leases->begin(); l != deleted_leases->end(); ++l) {
1234  }
1235 
1236  // Lease updates for new allocations and updated leases.
1237  for (auto l = leases->begin(); l != leases->end(); ++l) {
1239  }
1240 
1241  continue;
1242  }
1243 
1244  // Check if the lease update should be sent to the server. If we're in
1245  // the partner-down state we don't send lease updates to the partner.
1246  if (!shouldSendLeaseUpdates(conf)) {
1247  // If we decide to not send the lease updates to an active partner, we
1248  // should make a record of it in the communication state. The partner
1249  // can check if there were any unsent lease updates when he determines
1250  // whether it should synchronize its database or not when it recovers
1251  // from the partner-down state.
1252  if (conf->getRole() != HAConfig::PeerConfig::BACKUP) {
1253  communication_state_->increaseUnsentUpdateCount();
1254  }
1255  continue;
1256  }
1257 
1258  // If we're contacting a backup server from which we don't expect a
1259  // response prior to responding to the DHCP client we don't count
1260  // it.
1261  if (config_->amWaitingBackupAck() || (conf->getRole() != HAConfig::PeerConfig::BACKUP)) {
1262  ++sent_num;
1263  }
1264 
1265  // Send new/updated leases and deleted leases in one command.
1266  asyncSendLeaseUpdate(query, conf, CommandCreator::createLease6BulkApply(leases, deleted_leases),
1267  parking_lot);
1268  }
1269 
1270  return (sent_num);
1271 }
1272 
1273 template<typename QueryPtrType>
1274 bool
1276  const ParkingLotHandlePtr& parking_lot) {
1277  if (MultiThreadingMgr::instance().getMode()) {
1278  std::lock_guard<std::mutex> lock(mutex_);
1279  return (leaseUpdateCompleteInternal(query, parking_lot));
1280  } else {
1281  return (leaseUpdateCompleteInternal(query, parking_lot));
1282  }
1283 }
1284 
1285 template<typename QueryPtrType>
1286 bool
1287 HAService::leaseUpdateCompleteInternal(QueryPtrType& query,
1288  const ParkingLotHandlePtr& parking_lot) {
1289  auto it = pending_requests_.find(query);
1290 
1291  // If there are no more pending requests for this query, let's unpark
1292  // the DHCP packet.
1293  if (it == pending_requests_.end() || (--pending_requests_[query] <= 0)) {
1294  parking_lot->unpark(query);
1295 
1296  // If we have unparked the packet we can clear pending requests for
1297  // this query.
1298  if (it != pending_requests_.end()) {
1299  pending_requests_.erase(it);
1300  }
1301  return (true);
1302  }
1303  return (false);
1304 }
1305 
1306 template<typename QueryPtrType>
1307 void
1308 HAService::updatePendingRequest(QueryPtrType& query) {
1309  if (MultiThreadingMgr::instance().getMode()) {
1310  std::lock_guard<std::mutex> lock(mutex_);
1311  updatePendingRequestInternal(query);
1312  } else {
1313  updatePendingRequestInternal(query);
1314  }
1315 }
1316 
1317 template<typename QueryPtrType>
1318 void
1319 HAService::updatePendingRequestInternal(QueryPtrType& query) {
1320  if (pending_requests_.count(query) == 0) {
1321  pending_requests_[query] = 1;
1322  } else {
1323  ++pending_requests_[query];
1324  }
1325 }
1326 
1327 template<typename QueryPtrType>
1328 void
1329 HAService::asyncSendLeaseUpdate(const QueryPtrType& query,
1330  const HAConfig::PeerConfigPtr& config,
1331  const ConstElementPtr& command,
1332  const ParkingLotHandlePtr& parking_lot) {
1333  // Create HTTP/1.1 request including our command.
1334  PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
1335  (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
1336  HostHttpHeader(config->getUrl().getStrippedHostname()));
1337  config->addBasicAuthHttpHeader(request);
1338  request->setBodyAsJson(command);
1339  request->finalize();
1340 
1341  // Response object should also be created because the HTTP client needs
1342  // to know the type of the expected response.
1343  HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
1344 
1345  // When possible we prefer to pass weak pointers to the queries, rather
1346  // than shared pointers, to avoid memory leaks in case cross reference
1347  // between the pointers.
1348  boost::weak_ptr<typename QueryPtrType::element_type> weak_query(query);
1349 
1350  // Schedule asynchronous HTTP request.
1351  client_->asyncSendRequest(config->getUrl(), config->getTlsContext(),
1352  request, response,
1353  [this, weak_query, parking_lot, config]
1354  (const boost::system::error_code& ec,
1355  const HttpResponsePtr& response,
1356  const std::string& error_str) {
1357  // Get the shared pointer of the query. The server should keep the
1358  // pointer to the query and then park it. Therefore, we don't really
1359  // expect it to be null. If it is null, something is really wrong.
1360  QueryPtrType query = weak_query.lock();
1361  if (!query) {
1362  isc_throw(Unexpected, "query is null while receiving response from"
1363  " HA peer. This is programmatic error");
1364  }
1365 
1366  // There are three possible groups of errors during the lease update.
1367  // One is the IO error causing issues in communication with the peer.
1368  // Another one is an HTTP parsing error. The last type of error is
1369  // when non-success error code is returned in the response carried
1370  // in the HTTP message or if the JSON response is otherwise broken.
1371 
1372  bool lease_update_success = true;
1373 
1374  // Handle first two groups of errors.
1375  if (ec || !error_str.empty()) {
1377  .arg(query->getLabel())
1378  .arg(config->getLogLabel())
1379  .arg(ec ? ec.message() : error_str);
1380 
1381  // Communication error, so let's drop parked packet. The DHCP
1382  // response will not be sent.
1383  lease_update_success = false;
1384 
1385  } else {
1386 
1387  // Handle third group of errors.
1388  try {
1389  int rcode = 0;
1390  auto args = verifyAsyncResponse(response, rcode);
1391  // In the v6 case the server may return a list of failed lease
1392  // updates and we should log them.
1393  logFailedLeaseUpdates(query, args);
1394 
1395  } catch (const std::exception& ex) {
1397  .arg(query->getLabel())
1398  .arg(config->getLogLabel())
1399  .arg(ex.what());
1400 
1401  // Error while doing an update. The DHCP response will not be sent.
1402  lease_update_success = false;
1403  }
1404  }
1405 
1406  // We don't care about the result of the lease update to the backup server.
1407  // It is a best effort update.
1408  if ((config->getRole() != HAConfig::PeerConfig::BACKUP) && !lease_update_success) {
1409  // If we were unable to communicate with the partner we set partner's
1410  // state as unavailable.
1411  communication_state_->setPartnerState("unavailable");
1412  }
1413 
1414  // It is possible to configure the server to not wait for a response from
1415  // the backup server before we unpark the packet and respond to the client.
1416  // Here we check if we're dealing with such situation.
1417  if (config_->amWaitingBackupAck() || (config->getRole() != HAConfig::PeerConfig::BACKUP)) {
1418  // We're expecting a response from the backup server or it is not
1419  // a backup server and the lease update was unsuccessful. In such
1420  // case the DHCP exchange fails.
1421  if (!lease_update_success) {
1422  parking_lot->drop(query);
1423  }
1424  } else {
1425  // This was a response from the backup server and we're configured to
1426  // not wait for their acknowledgments, so there is nothing more to do.
1427  return;
1428  }
1429 
1430  if (leaseUpdateComplete(query, parking_lot)) {
1431  // If we have finished sending the lease updates we need to run the
1432  // state machine until the state machine finds that additional events
1433  // are required, such as next heartbeat or a lease update. The runModel()
1434  // may transition to another state, schedule asynchronous tasks etc.
1435  // Then it returns control to the DHCP server.
1437  }
1438  },
1440  std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
1441  std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
1442  std::bind(&HAService::clientCloseHandler, this, ph::_1)
1443  );
1444 
1445  // The number of pending requests is the number of requests for which we
1446  // expect an acknowledgment prior to responding to the DHCP clients. If
1447  // we're configured to wait for the acks from the backups or it is not
1448  // a backup increase the number of pending requests.
1449  if (config_->amWaitingBackupAck() || (config->getRole() != HAConfig::PeerConfig::BACKUP)) {
1450  // Request scheduled, so update the request counters for the query.
1451  updatePendingRequest(query);
1452  }
1453 }
1454 
1455 bool
1457  // Never send lease updates if they are administratively disabled.
1458  if (!config_->amSendingLeaseUpdates()) {
1459  return (false);
1460  }
1461 
1462  // Always send updates to the backup server.
1463  if (peer_config->getRole() == HAConfig::PeerConfig::BACKUP) {
1464  return (true);
1465  }
1466 
1467  // Never send updates if this is a backup server.
1468  if (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::BACKUP) {
1469  return (false);
1470  }
1471 
1472  // In other case, whether we send lease updates or not depends on our
1473  // state.
1474  switch (getCurrState()) {
1475  case HA_HOT_STANDBY_ST:
1476  case HA_LOAD_BALANCING_ST:
1478  return (true);
1479 
1480  default:
1481  ;
1482  }
1483 
1484  return (false);
1485 }
1486 
1487 bool
1489  if (!config_->amSendingLeaseUpdates()) {
1490  return (false);
1491  }
1492 
1493  if (peer_config->getRole() == HAConfig::PeerConfig::BACKUP) {
1494  return (false);
1495  }
1496 
1498 }
1499 
1500 void
1502  const ConstElementPtr& args) const {
1503  // If there are no arguments, it means that the update was successful.
1504  if (!args || (args->getType() != Element::map)) {
1505  return;
1506  }
1507 
1508  // Instead of duplicating the code between the failed-deleted-leases and
1509  // failed-leases, let's just have one function that does it for both.
1510  auto log_proc = [](const PktPtr query, const ConstElementPtr& args,
1511  const std::string& param_name, const log::MessageID& mesid) {
1512 
1513  // Check if there are any failed leases.
1514  auto failed_leases = args->get(param_name);
1515 
1516  // The failed leases must be a list.
1517  if (failed_leases && (failed_leases->getType() == Element::list)) {
1518  // Go over the failed leases and log each of them.
1519  for (int i = 0; i < failed_leases->size(); ++i) {
1520  auto lease = failed_leases->get(i);
1521  if (lease->getType() == Element::map) {
1522 
1523  // ip-address
1524  auto ip_address = lease->get("ip-address");
1525 
1526  // lease type
1527  auto lease_type = lease->get("type");
1528 
1529  // error-message
1530  auto error_message = lease->get("error-message");
1531 
1532  LOG_INFO(ha_logger, mesid)
1533  .arg(query->getLabel())
1534  .arg(lease_type && (lease_type->getType() == Element::string) ?
1535  lease_type->stringValue() : "(unknown)")
1536  .arg(ip_address && (ip_address->getType() == Element::string) ?
1537  ip_address->stringValue() : "(unknown)")
1538  .arg(error_message && (error_message->getType() == Element::string) ?
1539  error_message->stringValue() : "(unknown)");
1540  }
1541  }
1542  }
1543  };
1544 
1545  // Process "failed-deleted-leases"
1546  log_proc(query, args, "failed-deleted-leases", HA_LEASE_UPDATE_DELETE_FAILED_ON_PEER);
1547 
1548  // Process "failed-leases".
1549  log_proc(query, args, "failed-leases", HA_LEASE_UPDATE_CREATE_UPDATE_FAILED_ON_PEER);
1550 }
1551 
1554  ElementPtr ha_servers = Element::createMap();
1555 
1556  // Local part
1557  ElementPtr local = Element::createMap();
1559  role = config_->getThisServerConfig()->getRole();
1560  std::string role_txt = HAConfig::PeerConfig::roleToString(role);
1561  local->set("role", Element::create(role_txt));
1562  int state = getCurrState();
1563  try {
1564  local->set("state", Element::create(stateToString(state)));
1565 
1566  } catch (...) {
1567  // Empty string on error.
1568  local->set("state", Element::create(std::string()));
1569  }
1570  std::set<std::string> scopes = query_filter_.getServedScopes();
1571  ElementPtr list = Element::createList();
1572  for (std::string scope : scopes) {
1573  list->add(Element::create(scope));
1574  }
1575  local->set("scopes", list);
1576  ha_servers->set("local", local);
1577 
1578  // Do not include remote server information if this is a backup server or
1579  // we're in the passive-backup mode.
1580  if ((config_->getHAMode() == HAConfig::PASSIVE_BACKUP) ||
1581  (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::BACKUP)) {
1582  return (ha_servers);
1583  }
1584 
1585  // Remote part
1586  ElementPtr remote = communication_state_->getReport();
1587 
1588  try {
1589  role = config_->getFailoverPeerConfig()->getRole();
1590  std::string role_txt = HAConfig::PeerConfig::roleToString(role);
1591  remote->set("role", Element::create(role_txt));
1592 
1593  } catch (...) {
1594  remote->set("role", Element::create(std::string()));
1595  }
1596  ha_servers->set("remote", remote);
1597 
1598  return (ha_servers);
1599 }
1600 
1603  ElementPtr arguments = Element::createMap();
1604  std::string state_label = getState(getCurrState())->getLabel();
1605  arguments->set("state", Element::create(state_label));
1606 
1607  std::string date_time = HttpDateTime().rfc1123Format();
1608  arguments->set("date-time", Element::create(date_time));
1609 
1610  auto scopes = query_filter_.getServedScopes();
1611  ElementPtr scopes_list = Element::createList();
1612  for (auto scope : scopes) {
1613  scopes_list->add(Element::create(scope));
1614  }
1615  arguments->set("scopes", scopes_list);
1616 
1617  arguments->set("unsent-update-count",
1618  Element::create(static_cast<int64_t>(communication_state_->getUnsentUpdateCount())));
1619 
1620  return (createAnswer(CONTROL_RESULT_SUCCESS, "HA peer status returned.",
1621  arguments));
1622 }
1623 
1626  if (getCurrState() == HA_WAITING_ST) {
1627  return (createAnswer(CONTROL_RESULT_SUCCESS, "HA state machine already in WAITING state."));
1628  }
1630  runModel(NOP_EVT);
1631  return (createAnswer(CONTROL_RESULT_SUCCESS, "HA state machine reset."));
1632 }
1633 
1634 void
1636  HAConfig::PeerConfigPtr partner_config = config_->getFailoverPeerConfig();
1637 
1638  // If the sync_complete_notified_ is true it means that the partner
1639  // notified us that it had completed lease database synchronization.
1640  // We confirm that the partner is operational by sending the heartbeat
1641  // to it. Regardless if the partner responds to our heartbeats or not,
1642  // we should clear this flag. But, since we need the current value in
1643  // the async call handler, we save it in the local variable before
1644  // clearing it.
1645  bool sync_complete_notified = sync_complete_notified_;
1646  sync_complete_notified_ = false;
1647 
1648  // Create HTTP/1.1 request including our command.
1649  PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
1650  (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
1651  HostHttpHeader(partner_config->getUrl().getStrippedHostname()));
1652  partner_config->addBasicAuthHttpHeader(request);
1653  request->setBodyAsJson(CommandCreator::createHeartbeat(server_type_));
1654  request->finalize();
1655 
1656  // Response object should also be created because the HTTP client needs
1657  // to know the type of the expected response.
1658  HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
1659 
1660  // Schedule asynchronous HTTP request.
1661  client_->asyncSendRequest(partner_config->getUrl(),
1662  partner_config->getTlsContext(),
1663  request, response,
1664  [this, partner_config, sync_complete_notified]
1665  (const boost::system::error_code& ec,
1666  const HttpResponsePtr& response,
1667  const std::string& error_str) {
1668 
1669  // There are three possible groups of errors during the heartbeat.
1670  // One is the IO error causing issues in communication with the peer.
1671  // Another one is an HTTP parsing error. The last type of error is
1672  // when non-success error code is returned in the response carried
1673  // in the HTTP message or if the JSON response is otherwise broken.
1674 
1675  bool heartbeat_success = true;
1676 
1677  // Handle first two groups of errors.
1678  if (ec || !error_str.empty()) {
1680  .arg(partner_config->getLogLabel())
1681  .arg(ec ? ec.message() : error_str);
1682  heartbeat_success = false;
1683 
1684  } else {
1685 
1686  // Handle third group of errors.
1687  try {
1688  // Response must contain arguments and the arguments must
1689  // be a map.
1690  int rcode = 0;
1691  ConstElementPtr args = verifyAsyncResponse(response, rcode);
1692  if (!args || args->getType() != Element::map) {
1693  isc_throw(CtrlChannelError, "returned arguments in the response"
1694  " must be a map");
1695  }
1696  // Response must include partner's state.
1697  ConstElementPtr state = args->get("state");
1698  if (!state || state->getType() != Element::string) {
1699  isc_throw(CtrlChannelError, "server state not returned in response"
1700  " to a ha-heartbeat command or it is not a string");
1701  }
1702  // Remember the partner's state. This may throw if the returned
1703  // state is invalid.
1704  communication_state_->setPartnerState(state->stringValue());
1705 
1706  ConstElementPtr date_time = args->get("date-time");
1707  if (!date_time || date_time->getType() != Element::string) {
1708  isc_throw(CtrlChannelError, "date-time not returned in response"
1709  " to a ha-heartbeat command or it is not a string");
1710  }
1711  // Note the time returned by the partner to calculate the clock skew.
1712  communication_state_->setPartnerTime(date_time->stringValue());
1713 
1714  // Remember the scopes served by the partner.
1715  try {
1716  auto scopes = args->get("scopes");
1717  communication_state_->setPartnerScopes(scopes);
1718 
1719  } catch (...) {
1720  // We don't want to fail if the scopes are missing because
1721  // this would be incompatible with old HA hook library
1722  // versions. We may make it mandatory one day, but during
1723  // upgrades of existing HA setup it would be a real issue
1724  // if we failed here.
1725  }
1726 
1727  // unsent-update-count was not present in earlier HA versions.
1728  // Let's check if the partner has sent the parameter. We initialized
1729  // the counter to 0, and it remains 0 if the partner doesn't send it.
1730  // It effectively means that we don't track partner's unsent updates
1731  // as in the earlier HA versions.
1732  auto unsent_update_count = args->get("unsent-update-count");
1733  if (unsent_update_count) {
1734  if (unsent_update_count->getType() != Element::integer) {
1735  isc_throw(CtrlChannelError, "unsent-update-count returned in"
1736  " the ha-heartbeat response is not an integer");
1737  }
1738  communication_state_->setPartnerUnsentUpdateCount(static_cast<uint64_t>
1739  (unsent_update_count->intValue()));
1740  }
1741 
1742  } catch (const std::exception& ex) {
1744  .arg(partner_config->getLogLabel())
1745  .arg(ex.what());
1746  heartbeat_success = false;
1747  }
1748  }
1749 
1750  // If heartbeat was successful, let's mark the connection with the
1751  // peer as healthy.
1752  if (heartbeat_success) {
1753  communication_state_->poke();
1754 
1755  } else {
1756  // We were unable to retrieve partner's state, so let's mark it
1757  // as unavailable.
1758  communication_state_->setPartnerState("unavailable");
1759  // Log if the communication is interrupted.
1760  if (communication_state_->isCommunicationInterrupted()) {
1762  .arg(partner_config->getName());
1763  }
1764  }
1765 
1766  startHeartbeat();
1767  // Even though the partner notified us about the synchronization completion,
1768  // we still can't communicate with the partner. Let's continue serving
1769  // the clients until the link is fixed.
1770  if (sync_complete_notified && !heartbeat_success) {
1772  }
1773  // Whatever the result of the heartbeat was, the state machine needs
1774  // to react to this. Let's run the state machine until the state machine
1775  // finds that some new events are required, i.e. next heartbeat or
1776  // lease update. The runModel() may transition to another state, schedule
1777  // asynchronous tasks etc. Then it returns control to the DHCP server.
1779  },
1781  std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
1782  std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
1783  std::bind(&HAService::clientCloseHandler, this, ph::_1)
1784  );
1785 }
1786 
1787 void
1789  if (!communication_state_->isHeartbeatRunning()) {
1790  startHeartbeat();
1791  }
1792 }
1793 
1794 void
1796  if (config_->getHeartbeatDelay() > 0) {
1797  communication_state_->startHeartbeat(config_->getHeartbeatDelay(),
1798  std::bind(&HAService::asyncSendHeartbeat,
1799  this));
1800  }
1801 }
1802 
1803 void
1805  const std::string& server_name,
1806  const unsigned int max_period,
1807  PostRequestCallback post_request_action) {
1808  HAConfig::PeerConfigPtr remote_config = config_->getPeerConfig(server_name);
1809 
1810  // Create HTTP/1.1 request including our command.
1811  PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
1812  (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
1813  HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
1814 
1815  remote_config->addBasicAuthHttpHeader(request);
1816  request->setBodyAsJson(CommandCreator::createDHCPDisable(max_period,
1817  server_type_));
1818  request->finalize();
1819 
1820  // Response object should also be created because the HTTP client needs
1821  // to know the type of the expected response.
1822  HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
1823 
1824  // Schedule asynchronous HTTP request.
1825  http_client.asyncSendRequest(remote_config->getUrl(),
1826  remote_config->getTlsContext(),
1827  request, response,
1828  [this, remote_config, post_request_action]
1829  (const boost::system::error_code& ec,
1830  const HttpResponsePtr& response,
1831  const std::string& error_str) {
1832 
1833  // There are three possible groups of errors during the heartbeat.
1834  // One is the IO error causing issues in communication with the peer.
1835  // Another one is an HTTP parsing error. The last type of error is
1836  // when non-success error code is returned in the response carried
1837  // in the HTTP message or if the JSON response is otherwise broken.
1838 
1839  int rcode = 0;
1840  std::string error_message;
1841 
1842  // Handle first two groups of errors.
1843  if (ec || !error_str.empty()) {
1844  error_message = (ec ? ec.message() : error_str);
1846  .arg(remote_config->getLogLabel())
1847  .arg(error_message);
1848 
1849  } else {
1850 
1851  // Handle third group of errors.
1852  try {
1853  static_cast<void>(verifyAsyncResponse(response, rcode));
1854 
1855  } catch (const std::exception& ex) {
1856  error_message = ex.what();
1858  .arg(remote_config->getLogLabel())
1859  .arg(error_message);
1860  }
1861  }
1862 
1863  // If there was an error communicating with the partner, mark the
1864  // partner as unavailable.
1865  if (!error_message.empty()) {
1866  communication_state_->setPartnerState("unavailable");
1867  }
1868 
1869  // Invoke post request action if it was specified.
1870  if (post_request_action) {
1871  post_request_action(error_message.empty(),
1872  error_message,
1873  rcode);
1874  }
1875  },
1877  std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
1878  std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
1879  std::bind(&HAService::clientCloseHandler, this, ph::_1)
1880  );
1881 }
1882 
1883 void
1885  const std::string& server_name,
1886  PostRequestCallback post_request_action) {
1887  HAConfig::PeerConfigPtr remote_config = config_->getPeerConfig(server_name);
1888 
1889  // Create HTTP/1.1 request including our command.
1890  PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
1891  (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
1892  HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
1893  remote_config->addBasicAuthHttpHeader(request);
1894  request->setBodyAsJson(CommandCreator::createDHCPEnable(server_type_));
1895  request->finalize();
1896 
1897  // Response object should also be created because the HTTP client needs
1898  // to know the type of the expected response.
1899  HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
1900 
1901  // Schedule asynchronous HTTP request.
1902  http_client.asyncSendRequest(remote_config->getUrl(),
1903  remote_config->getTlsContext(),
1904  request, response,
1905  [this, remote_config, post_request_action]
1906  (const boost::system::error_code& ec,
1907  const HttpResponsePtr& response,
1908  const std::string& error_str) {
1909 
1910  // There are three possible groups of errors during the heartbeat.
1911  // One is the IO error causing issues in communication with the peer.
1912  // Another one is an HTTP parsing error. The last type of error is
1913  // when non-success error code is returned in the response carried
1914  // in the HTTP message or if the JSON response is otherwise broken.
1915 
1916  int rcode = 0;
1917  std::string error_message;
1918 
1919  // Handle first two groups of errors.
1920  if (ec || !error_str.empty()) {
1921  error_message = (ec ? ec.message() : error_str);
1923  .arg(remote_config->getLogLabel())
1924  .arg(error_message);
1925 
1926  } else {
1927 
1928  // Handle third group of errors.
1929  try {
1930  static_cast<void>(verifyAsyncResponse(response, rcode));
1931 
1932  } catch (const std::exception& ex) {
1933  error_message = ex.what();
1935  .arg(remote_config->getLogLabel())
1936  .arg(error_message);
1937  }
1938  }
1939 
1940  // If there was an error communicating with the partner, mark the
1941  // partner as unavailable.
1942  if (!error_message.empty()) {
1943  communication_state_->setPartnerState("unavailable");
1944  }
1945 
1946  // Invoke post request action if it was specified.
1947  if (post_request_action) {
1948  post_request_action(error_message.empty(),
1949  error_message,
1950  rcode);
1951  }
1952  },
1954  std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
1955  std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
1956  std::bind(&HAService::clientCloseHandler, this, ph::_1)
1957  );
1958 }
1959 
1960 void
1962  network_state_->disableService(NetworkState::Origin::HA_COMMAND);
1963 }
1964 
1965 void
1967  network_state_->enableService(NetworkState::Origin::HA_COMMAND);
1968 }
1969 
1970 void
1972  PostSyncCallback null_action;
1973 
1974  // Timeout is configured in milliseconds. Need to convert to seconds.
1975  unsigned int dhcp_disable_timeout =
1976  static_cast<unsigned int>(config_->getSyncTimeout() / 1000);
1977  if (dhcp_disable_timeout == 0) {
1978  // Ensure that we always use at least 1 second timeout.
1979  dhcp_disable_timeout = 1;
1980  }
1981 
1982  asyncSyncLeases(*client_, config_->getFailoverPeerConfig()->getName(),
1983  dhcp_disable_timeout, LeasePtr(), null_action);
1984 }
1985 
1986 void
1988  const std::string& server_name,
1989  const unsigned int max_period,
1990  const dhcp::LeasePtr& last_lease,
1991  PostSyncCallback post_sync_action,
1992  const bool dhcp_disabled) {
1993  // Synchronization starts with a command to disable DHCP service of the
1994  // peer from which we're fetching leases. We don't want the other server
1995  // to allocate new leases while we fetch from it. The DHCP service will
1996  // be disabled for a certain amount of time and will be automatically
1997  // re-enabled if we die during the synchronization.
1998  asyncDisableDHCPService(http_client, server_name, max_period,
1999  [this, &http_client, server_name, max_period, last_lease,
2000  post_sync_action, dhcp_disabled]
2001  (const bool success, const std::string& error_message, const int) {
2002 
2003  // If we have successfully disabled the DHCP service on the peer,
2004  // we can start fetching the leases.
2005  if (success) {
2006  // The last argument indicates that disabling the DHCP
2007  // service on the partner server was successful.
2008  asyncSyncLeasesInternal(http_client, server_name, max_period,
2009  last_lease, post_sync_action, true);
2010 
2011  } else {
2012  post_sync_action(success, error_message, dhcp_disabled);
2013  }
2014  });
2015 }
2016 
2017 void
2019  const std::string& server_name,
2020  const unsigned int max_period,
2021  const dhcp::LeasePtr& last_lease,
2022  PostSyncCallback post_sync_action,
2023  const bool dhcp_disabled) {
2024 
2025  HAConfig::PeerConfigPtr partner_config = config_->getFailoverPeerConfig();
2026 
2027  // Create HTTP/1.1 request including our command.
2028  PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2029  (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
2030  HostHttpHeader(partner_config->getUrl().getStrippedHostname()));
2031  partner_config->addBasicAuthHttpHeader(request);
2033  request->setBodyAsJson(CommandCreator::createLease4GetPage(
2034  boost::dynamic_pointer_cast<Lease4>(last_lease), config_->getSyncPageLimit()));
2035 
2036  } else {
2037  request->setBodyAsJson(CommandCreator::createLease6GetPage(
2038  boost::dynamic_pointer_cast<Lease6>(last_lease), config_->getSyncPageLimit()));
2039  }
2040  request->finalize();
2041 
2042  // Response object should also be created because the HTTP client needs
2043  // to know the type of the expected response.
2044  HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2045 
2046  // Schedule asynchronous HTTP request.
2047  http_client.asyncSendRequest(partner_config->getUrl(),
2048  partner_config->getTlsContext(),
2049  request, response,
2050  [this, partner_config, post_sync_action, &http_client, server_name,
2051  max_period, dhcp_disabled]
2052  (const boost::system::error_code& ec,
2053  const HttpResponsePtr& response,
2054  const std::string& error_str) {
2055 
2056  // Holds last lease received on the page of leases. If the last
2057  // page was hit, this value remains null.
2058  LeasePtr last_lease;
2059 
2060  // There are three possible groups of errors during the heartbeat.
2061  // One is the IO error causing issues in communication with the peer.
2062  // Another one is an HTTP parsing error. The last type of error is
2063  // when non-success error code is returned in the response carried
2064  // in the HTTP message or if the JSON response is otherwise broken.
2065 
2066  std::string error_message;
2067 
2068  // Handle first two groups of errors.
2069  if (ec || !error_str.empty()) {
2070  error_message = (ec ? ec.message() : error_str);
2072  .arg(partner_config->getLogLabel())
2073  .arg(error_message);
2074 
2075  } else {
2076  // Handle third group of errors.
2077  try {
2078  int rcode = 0;
2079  ConstElementPtr args = verifyAsyncResponse(response, rcode);
2080 
2081  // Arguments must be a map.
2082  if (args && (args->getType() != Element::map)) {
2084  "arguments in the received response must be a map");
2085  }
2086 
2087  ConstElementPtr leases = args->get("leases");
2088  if (!leases || (leases->getType() != Element::list)) {
2090  "server response does not contain leases argument or this"
2091  " argument is not a list");
2092  }
2093 
2094  // Iterate over the leases and update the database as appropriate.
2095  const auto& leases_element = leases->listValue();
2096 
2098  .arg(leases_element.size())
2099  .arg(server_name);
2100 
2101  for (auto l = leases_element.begin(); l != leases_element.end(); ++l) {
2102  try {
2103 
2105  Lease4Ptr lease = Lease4::fromElement(*l);
2106 
2107  // Check if there is such lease in the database already.
2108  Lease4Ptr existing_lease = LeaseMgrFactory::instance().getLease4(lease->addr_);
2109  if (!existing_lease) {
2110  // There is no such lease, so let's add it.
2111  LeaseMgrFactory::instance().addLease(lease);
2112 
2113  } else if (existing_lease->cltt_ < lease->cltt_) {
2114  // If the existing lease is older than the fetched lease, update
2115  // the lease in our local database.
2116  // Update lease current expiration time with value received from the
2117  // database. Some database backends reject operations on the lease if
2118  // the current expiration time value does not match what is stored.
2119  Lease::syncCurrentExpirationTime(*existing_lease, *lease);
2120  LeaseMgrFactory::instance().updateLease4(lease);
2121 
2122  } else {
2124  .arg(lease->addr_.toText())
2125  .arg(lease->subnet_id_);
2126  }
2127 
2128  // If we're not on the last page and we're processing final lease on
2129  // this page, let's record the lease as input to the next
2130  // lease4-get-page command.
2131  if ((leases_element.size() >= config_->getSyncPageLimit()) &&
2132  (l + 1 == leases_element.end())) {
2133  last_lease = boost::dynamic_pointer_cast<Lease>(lease);
2134  }
2135 
2136  } else {
2137  Lease6Ptr lease = Lease6::fromElement(*l);
2138 
2139  // Check if there is such lease in the database already.
2140  Lease6Ptr existing_lease = LeaseMgrFactory::instance().getLease6(lease->type_,
2141  lease->addr_);
2142  if (!existing_lease) {
2143  // There is no such lease, so let's add it.
2144  LeaseMgrFactory::instance().addLease(lease);
2145 
2146  } else if (existing_lease->cltt_ < lease->cltt_) {
2147  // If the existing lease is older than the fetched lease, update
2148  // the lease in our local database.
2149  // Update lease current expiration time with value received from the
2150  // database. Some database backends reject operations on the lease if
2151  // the current expiration time value does not match what is stored.
2152  Lease::syncCurrentExpirationTime(*existing_lease, *lease);
2153  LeaseMgrFactory::instance().updateLease6(lease);
2154 
2155  } else {
2157  .arg(lease->addr_.toText())
2158  .arg(lease->subnet_id_);
2159  }
2160 
2161  // If we're not on the last page and we're processing final lease on
2162  // this page, let's record the lease as input to the next
2163  // lease6-get-page command.
2164  if ((leases_element.size() >= config_->getSyncPageLimit()) &&
2165  (l + 1 == leases_element.end())) {
2166  last_lease = boost::dynamic_pointer_cast<Lease>(lease);
2167  }
2168  }
2169 
2170  } catch (const std::exception& ex) {
2172  .arg((*l)->str())
2173  .arg(ex.what());
2174  }
2175  }
2176 
2177  } catch (const std::exception& ex) {
2178  error_message = ex.what();
2180  .arg(partner_config->getLogLabel())
2181  .arg(error_message);
2182  }
2183  }
2184 
2185  // If there was an error communicating with the partner, mark the
2186  // partner as unavailable.
2187  if (!error_message.empty()) {
2188  communication_state_->setPartnerState("unavailable");
2189 
2190  } else if (last_lease) {
2191  // This indicates that there are more leases to be fetched.
2192  // Therefore, we have to send another leaseX-get-page command.
2193  asyncSyncLeases(http_client, server_name, max_period, last_lease,
2194  post_sync_action, dhcp_disabled);
2195  return;
2196  }
2197 
2198  // Invoke post synchronization action if it was specified.
2199  if (post_sync_action) {
2200  post_sync_action(error_message.empty(),
2201  error_message,
2202  dhcp_disabled);
2203  }
2204  },
2205  HttpClient::RequestTimeout(config_->getSyncTimeout()),
2206  std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
2207  std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
2208  std::bind(&HAService::clientCloseHandler, this, ph::_1)
2209  );
2210 
2211 }
2212 
2214 HAService::processSynchronize(const std::string& server_name,
2215  const unsigned int max_period) {
2216  std::string answer_message;
2217  int sync_status = synchronize(answer_message, server_name, max_period);
2218  return (createAnswer(sync_status, answer_message));
2219 }
2220 
2221 int
2222 HAService::synchronize(std::string& status_message, const std::string& server_name,
2223  const unsigned int max_period) {
2224  IOService io_service;
2225  HttpClient client(io_service);
2226 
2227  asyncSyncLeases(client, server_name, max_period, Lease4Ptr(),
2228  [&](const bool success, const std::string& error_message,
2229  const bool dhcp_disabled) {
2230  // If there was a fatal error while fetching the leases, let's
2231  // log an error message so as it can be included in the response
2232  // to the controlling client.
2233  if (!success) {
2234  status_message = error_message;
2235  }
2236 
2237  // Whether or not there was an error while fetching the leases,
2238  // we need to re-enable the DHCP service on the peer if the
2239  // DHCP service was disabled in the course of synchronization.
2240  if (dhcp_disabled) {
2241  // If the synchronization was completed successfully let's
2242  // try to send the ha-sync-complete-notify command to the
2243  // partner.
2244  if (success) {
2245  asyncSyncCompleteNotify(client, server_name,
2246  [&](const bool success,
2247  const std::string& error_message,
2248  const int rcode) {
2249  // This command may not be supported by the partner when it
2250  // runs an older Kea version. In that case, send the dhcp-enable
2251  // command as in previous Kea version.
2252  if (rcode == CONTROL_RESULT_COMMAND_UNSUPPORTED) {
2253  asyncEnableDHCPService(client, server_name,
2254  [&](const bool success,
2255  const std::string& error_message,
2256  const int) {
2257  // It is possible that we have already recorded an error
2258  // message while synchronizing the lease database. Don't
2259  // override the existing error message.
2260  if (!success && status_message.empty()) {
2261  status_message = error_message;
2262  }
2263 
2264  // The synchronization process is completed, so let's break
2265  // the IO service so as we can return the response to the
2266  // controlling client.
2267  io_service.stop();
2268  });
2269 
2270  } else {
2271  // ha-sync-complete-notify command was delivered to the partner.
2272  // The synchronization process ends here.
2273  if (!success && status_message.empty()) {
2274  status_message = error_message;
2275  }
2276 
2277  io_service.stop();
2278  }
2279  });
2280 
2281  } else {
2282  // Synchronization was unsuccessful. Send the dhcp-enable command to
2283  // re-enable the DHCP service. Note, that we don't send the
2284  // ha-sync-complete-notify command in this case. It is only sent in
2285  // the case when synchronization ends successfully.
2286  asyncEnableDHCPService(client, server_name,
2287  [&](const bool success,
2288  const std::string& error_message,
2289  const int) {
2290  if (!success && status_message.empty()) {
2291  status_message = error_message;
2292  }
2293 
2294  // The synchronization process is completed, so let's break
2295  // the IO service so as we can return the response to the
2296  // controlling client.
2297  io_service.stop();
2298 
2299  });
2300  }
2301 
2302  } else {
2303  // Also stop IO service if there is no need to enable DHCP
2304  // service.
2305  io_service.stop();
2306  }
2307  });
2308 
2309  LOG_INFO(ha_logger, HA_SYNC_START).arg(server_name);
2310 
2311  // Measure duration of the synchronization.
2312  Stopwatch stopwatch;
2313 
2314  // Run the IO service until it is stopped by any of the callbacks. This
2315  // makes it synchronous.
2316  io_service.run();
2317 
2318  // End measuring duration.
2319  stopwatch.stop();
2320 
2321  // If an error message has been recorded, return an error to the controlling
2322  // client.
2323  if (!status_message.empty()) {
2325 
2327  .arg(server_name)
2328  .arg(status_message);
2329 
2330  return (CONTROL_RESULT_ERROR);
2331 
2332  }
2333 
2334  // Everything was fine, so let's return a success.
2335  status_message = "Lease database synchronization complete.";
2337 
2339  .arg(server_name)
2340  .arg(stopwatch.logFormatLastDuration());
2341 
2342  return (CONTROL_RESULT_SUCCESS);
2343 }
2344 
2345 void
2347  const HAConfig::PeerConfigPtr& config,
2348  PostRequestCallback post_request_action) {
2349  if (lease_update_backlog_.size() == 0) {
2350  post_request_action(true, "", CONTROL_RESULT_SUCCESS);
2351  return;
2352  }
2353 
2354  ConstElementPtr command;
2357  Lease4Ptr lease = boost::dynamic_pointer_cast<Lease4>(lease_update_backlog_.pop(op_type));
2358  if (op_type == LeaseUpdateBacklog::ADD) {
2359  command = CommandCreator::createLease4Update(*lease);
2360  } else {
2361  command = CommandCreator::createLease4Delete(*lease);
2362  }
2363 
2364  } else {
2366  }
2367 
2368  // Create HTTP/1.1 request including our command.
2369  PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2370  (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
2371  HostHttpHeader(config->getUrl().getStrippedHostname()));
2372  config->addBasicAuthHttpHeader(request);
2373  request->setBodyAsJson(command);
2374  request->finalize();
2375 
2376  // Response object should also be created because the HTTP client needs
2377  // to know the type of the expected response.
2378  HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2379 
2380  http_client.asyncSendRequest(config->getUrl(), config->getTlsContext(),
2381  request, response,
2382  [this, &http_client, config, post_request_action]
2383  (const boost::system::error_code& ec,
2384  const HttpResponsePtr& response,
2385  const std::string& error_str) {
2386 
2387  int rcode = 0;
2388  std::string error_message;
2389 
2390  if (ec || !error_str.empty()) {
2391  error_message = (ec ? ec.message() : error_str);
2393  .arg(config->getLogLabel())
2394  .arg(ec ? ec.message() : error_str);
2395 
2396  } else {
2397  // Handle third group of errors.
2398  try {
2399  auto args = verifyAsyncResponse(response, rcode);
2400  } catch (const std::exception& ex) {
2401  error_message = ex.what();
2403  .arg(config->getLogLabel())
2404  .arg(ex.what());
2405  }
2406  }
2407 
2408  // Recursively send all outstanding lease updates or break when an
2409  // error occurs. In DHCPv6, this is a single iteration because we use
2410  // lease6-bulk-apply, which combines many lease updates in a single
2411  // transaction. In the case of DHCPv4, each update is sent in its own
2412  // transaction.
2413  if (error_message.empty()) {
2414  asyncSendLeaseUpdatesFromBacklog(http_client, config, post_request_action);
2415  } else {
2416  post_request_action(error_message.empty(), error_message, rcode);
2417  }
2418  });
2419 }
2420 
2421 bool
2423  auto num_updates = lease_update_backlog_.size();
2424  if (num_updates == 0) {
2426  return (true);
2427  }
2428 
2429  IOService io_service;
2430  HttpClient client(io_service);
2431  auto remote_config = config_->getFailoverPeerConfig();
2432  bool updates_successful = true;
2433 
2435  .arg(num_updates)
2436  .arg(remote_config->getName());
2437 
2438  asyncSendLeaseUpdatesFromBacklog(client, remote_config,
2439  [&](const bool success, const std::string&, const int) {
2440  io_service.stop();
2441  updates_successful = success;
2442  });
2443 
2444  // Measure duration of the updates.
2445  Stopwatch stopwatch;
2446 
2447  // Run the IO service until it is stopped by the callback. This makes it synchronous.
2448  io_service.run();
2449 
2450  // End measuring duration.
2451  stopwatch.stop();
2452 
2453  if (updates_successful) {
2455  .arg(remote_config->getName())
2456  .arg(stopwatch.logFormatLastDuration());
2457  }
2458 
2459  return (updates_successful);
2460 }
2461 
2462 void
2464  const HAConfig::PeerConfigPtr& config,
2465  PostRequestCallback post_request_action) {
2467 
2468  // Create HTTP/1.1 request including our command.
2469  PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2470  (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
2471  HostHttpHeader(config->getUrl().getStrippedHostname()));
2472  config->addBasicAuthHttpHeader(request);
2473  request->setBodyAsJson(command);
2474  request->finalize();
2475 
2476  // Response object should also be created because the HTTP client needs
2477  // to know the type of the expected response.
2478  HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2479 
2480  http_client.asyncSendRequest(config->getUrl(), config->getTlsContext(),
2481  request, response,
2482  [this, config, post_request_action]
2483  (const boost::system::error_code& ec,
2484  const HttpResponsePtr& response,
2485  const std::string& error_str) {
2486 
2487  int rcode = 0;
2488  std::string error_message;
2489 
2490  if (ec || !error_str.empty()) {
2491  error_message = (ec ? ec.message() : error_str);
2493  .arg(config->getLogLabel())
2494  .arg(ec ? ec.message() : error_str);
2495 
2496  } else {
2497  // Handle third group of errors.
2498  try {
2499  auto args = verifyAsyncResponse(response, rcode);
2500  } catch (const std::exception& ex) {
2501  error_message = ex.what();
2503  .arg(config->getLogLabel())
2504  .arg(ex.what());
2505  }
2506  }
2507 
2508  post_request_action(error_message.empty(), error_message, rcode);
2509  });
2510 }
2511 
2512 bool
2514  IOService io_service;
2515  HttpClient client(io_service);
2516  auto remote_config = config_->getFailoverPeerConfig();
2517  bool reset_successful = true;
2518 
2519  asyncSendHAReset(client, remote_config,
2520  [&](const bool success, const std::string&, const int) {
2521  io_service.stop();
2522  reset_successful = success;
2523  });
2524 
2525  // Run the IO service until it is stopped by the callback. This makes it synchronous.
2526  io_service.run();
2527 
2528  return (reset_successful);
2529 }
2530 
2532 HAService::processScopes(const std::vector<std::string>& scopes) {
2533  try {
2534  query_filter_.serveScopes(scopes);
2536 
2537  } catch (const std::exception& ex) {
2538  return (createAnswer(CONTROL_RESULT_ERROR, ex.what()));
2539  }
2540 
2541  return (createAnswer(CONTROL_RESULT_SUCCESS, "New HA scopes configured."));
2542 }
2543 
2546  if (unpause()) {
2547  return (createAnswer(CONTROL_RESULT_SUCCESS, "HA state machine continues."));
2548  }
2549  return (createAnswer(CONTROL_RESULT_SUCCESS, "HA state machine is not paused."));
2550 }
2551 
2554  if (cancel) {
2556  return (createAnswer(CONTROL_RESULT_ERROR, "Unable to cancel the"
2557  " maintenance for the server not in the"
2558  " in-maintenance state."));
2559  }
2560 
2563  runModel(NOP_EVT);
2564  return (createAnswer(CONTROL_RESULT_SUCCESS, "Server maintenance canceled."));
2565  }
2566 
2567  switch (getCurrState()) {
2568  case HA_BACKUP_ST:
2570  case HA_TERMINATED_ST:
2571  // The reason why we don't return an error result here is that we have to
2572  // have a way to distinguish between the errors caused by the communication
2573  // issues and the cases when there is no communication error but the server
2574  // is not allowed to enter the in-maintenance state. In the former case, the
2575  // partner would go to partner-down. In the case signaled by the special
2576  // result code entering the maintenance state is not allowed.
2578  "Unable to transition the server from the "
2579  + stateToString(getCurrState()) + " to"
2580  " in-maintenance state."));
2581  default:
2584  }
2585  return (createAnswer(CONTROL_RESULT_SUCCESS, "Server is in-maintenance state."));
2586 }
2587 
2590  switch (getCurrState()) {
2591  case HA_BACKUP_ST:
2592  case HA_IN_MAINTENANCE_ST:
2594  case HA_TERMINATED_ST:
2595  return (createAnswer(CONTROL_RESULT_ERROR, "Unable to transition the server from"
2596  " the " + stateToString(getCurrState()) + " to"
2597  " partner-in-maintenance state."));
2598  default:
2599  ;
2600  }
2601 
2602  HAConfig::PeerConfigPtr remote_config = config_->getFailoverPeerConfig();
2603 
2604  // Create HTTP/1.1 request including ha-maintenance-notify command
2605  // with the cancel flag set to false.
2606  PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2607  (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
2608  HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
2609  remote_config->addBasicAuthHttpHeader(request);
2610  request->setBodyAsJson(CommandCreator::createMaintenanceNotify(false, server_type_));
2611  request->finalize();
2612 
2613  // Response object should also be created because the HTTP client needs
2614  // to know the type of the expected response.
2615  HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2616 
2617  IOService io_service;
2618  HttpClient client(io_service);
2619 
2620  boost::system::error_code captured_ec;
2621  std::string captured_error_message;
2622  int captured_rcode = 0;
2623 
2624  // Schedule asynchronous HTTP request.
2625  client.asyncSendRequest(remote_config->getUrl(),
2626  remote_config->getTlsContext(),
2627  request, response,
2628  [this, remote_config, &io_service, &captured_ec, &captured_error_message,
2629  &captured_rcode]
2630  (const boost::system::error_code& ec,
2631  const HttpResponsePtr& response,
2632  const std::string& error_str) {
2633 
2634  io_service.stop();
2635 
2636  // There are three possible groups of errors. One is the IO error
2637  // causing issues in communication with the peer. Another one is
2638  // an HTTP parsing error. The last type of error is when non-success
2639  // error code is returned in the response carried in the HTTP message
2640  // or if the JSON response is otherwise broken.
2641 
2642  std::string error_message;
2643 
2644  // Handle first two groups of errors.
2645  if (ec || !error_str.empty()) {
2646  error_message = (ec ? ec.message() : error_str);
2648  .arg(remote_config->getLogLabel())
2649  .arg(error_message);
2650 
2651  } else {
2652 
2653  // Handle third group of errors.
2654  try {
2655  static_cast<void>(verifyAsyncResponse(response, captured_rcode));
2656 
2657  } catch (const std::exception& ex) {
2658  error_message = ex.what();
2660  .arg(remote_config->getLogLabel())
2661  .arg(error_message);
2662  }
2663  }
2664 
2665  // If there was an error communicating with the partner, mark the
2666  // partner as unavailable.
2667  if (!error_message.empty()) {
2668  communication_state_->setPartnerState("unavailable");
2669  }
2670 
2671  captured_ec = ec;
2672  captured_error_message = error_message;
2673  },
2675  std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
2676  std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
2677  std::bind(&HAService::clientCloseHandler, this, ph::_1)
2678  );
2679 
2680  // Run the IO service until it is stopped by any of the callbacks. This
2681  // makes it synchronous.
2682  io_service.run();
2683 
2684  // If there was a communication problem with the partner we assume that
2685  // the partner is already down while we receive this command.
2686  if (captured_ec || (captured_rcode == CONTROL_RESULT_ERROR)) {
2689  runModel(NOP_EVT);
2691  "Server is now in the partner-down state as its"
2692  " partner appears to be offline for maintenance."));
2693 
2694  } else if (captured_rcode == CONTROL_RESULT_SUCCESS) {
2695  // If the partner responded indicating no error it means that the
2696  // partner has been transitioned to the in-maintenance state. In that
2697  // case we transition to the partner-in-maintenance state.
2700  runModel(NOP_EVT);
2701 
2702  } else {
2703  // Partner server returned a special status code which means that it can't
2704  // transition to the partner-in-maintenance state.
2705  return (createAnswer(CONTROL_RESULT_ERROR, "Unable to transition to the"
2706  " partner-in-maintenance state. The partner server responded"
2707  " with the following message to the ha-maintenance-notify"
2708  " command: " + captured_error_message + "."));
2709 
2710  }
2711 
2713  "Server is now in the partner-in-maintenance state"
2714  " and its partner is in-maintenance state. The partner"
2715  " can be now safely shut down."));
2716 }
2717 
2721  return (createAnswer(CONTROL_RESULT_ERROR, "Unable to cancel maintenance"
2722  " request because the server is not in the"
2723  " partner-in-maintenance state."));
2724  }
2725 
2726  HAConfig::PeerConfigPtr remote_config = config_->getFailoverPeerConfig();
2727 
2728  // Create HTTP/1.1 request including ha-maintenance-notify command
2729  // with the cancel flag set to true.
2730  PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2731  (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
2732  HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
2733  remote_config->addBasicAuthHttpHeader(request);
2734  request->setBodyAsJson(CommandCreator::createMaintenanceNotify(true, server_type_));
2735  request->finalize();
2736 
2737  // Response object should also be created because the HTTP client needs
2738  // to know the type of the expected response.
2739  HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2740 
2741  IOService io_service;
2742  HttpClient client(io_service);
2743 
2744  std::string error_message;
2745 
2746  // Schedule asynchronous HTTP request.
2747  client.asyncSendRequest(remote_config->getUrl(),
2748  remote_config->getTlsContext(),
2749  request, response,
2750  [this, remote_config, &io_service, &error_message]
2751  (const boost::system::error_code& ec,
2752  const HttpResponsePtr& response,
2753  const std::string& error_str) {
2754 
2755  io_service.stop();
2756 
2757  // Handle first two groups of errors.
2758  if (ec || !error_str.empty()) {
2759  error_message = (ec ? ec.message() : error_str);
2761  .arg(remote_config->getLogLabel())
2762  .arg(error_message);
2763 
2764  } else {
2765 
2766  // Handle third group of errors.
2767  try {
2768  int rcode = 0;
2769  static_cast<void>(verifyAsyncResponse(response, rcode));
2770 
2771  } catch (const std::exception& ex) {
2772  error_message = ex.what();
2774  .arg(remote_config->getLogLabel())
2775  .arg(error_message);
2776  }
2777  }
2778 
2779  // If there was an error communicating with the partner, mark the
2780  // partner as unavailable.
2781  if (!error_message.empty()) {
2782  communication_state_->setPartnerState("unavailable");
2783  }
2784  },
2786  std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
2787  std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
2788  std::bind(&HAService::clientCloseHandler, this, ph::_1)
2789  );
2790 
2791  // Run the IO service until it is stopped by any of the callbacks. This
2792  // makes it synchronous.
2793  io_service.run();
2794 
2795  // There was an error in communication with the partner or the
2796  // partner was unable to revert its state.
2797  if (!error_message.empty()) {
2799  "Unable to cancel maintenance. The partner server responded"
2800  " with the following message to the ha-maintenance-notify"
2801  " command: " + error_message + "."));
2802  }
2803 
2804  // Successfully reverted partner's state. Let's also revert our state to the
2805  // previous one.
2808  runModel(NOP_EVT);
2809 
2811  "Server maintenance successfully canceled."));
2812 }
2813 
2814 void
2816  const std::string& server_name,
2817  PostRequestCallback post_request_action) {
2818  HAConfig::PeerConfigPtr remote_config = config_->getPeerConfig(server_name);
2819 
2820  // Create HTTP/1.1 request including our command.
2821  PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2822  (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
2823  HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
2824 
2825  remote_config->addBasicAuthHttpHeader(request);
2826  request->setBodyAsJson(CommandCreator::createSyncCompleteNotify(server_type_));
2827  request->finalize();
2828 
2829  // Response object should also be created because the HTTP client needs
2830  // to know the type of the expected response.
2831  HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2832 
2833  // Schedule asynchronous HTTP request.
2834  http_client.asyncSendRequest(remote_config->getUrl(),
2835  remote_config->getTlsContext(),
2836  request, response,
2837  [this, remote_config, post_request_action]
2838  (const boost::system::error_code& ec,
2839  const HttpResponsePtr& response,
2840  const std::string& error_str) {
2841 
2842  // There are three possible groups of errors. One is the IO error
2843  // causing issues in communication with the peer. Another one is an
2844  // HTTP parsing error. The last type of error is when non-success
2845  // error code is returned in the response carried in the HTTP message
2846  // or if the JSON response is otherwise broken.
2847 
2848  int rcode = 0;
2849  std::string error_message;
2850 
2851  // Handle first two groups of errors.
2852  if (ec || !error_str.empty()) {
2853  error_message = (ec ? ec.message() : error_str);
2855  .arg(remote_config->getLogLabel())
2856  .arg(error_message);
2857 
2858  } else {
2859 
2860  // Handle third group of errors.
2861  try {
2862  static_cast<void>(verifyAsyncResponse(response, rcode));
2863 
2864  } catch (const CommandUnsupportedError& ex) {
2866 
2867  } catch (const std::exception& ex) {
2868  error_message = ex.what();
2870  .arg(remote_config->getLogLabel())
2871  .arg(error_message);
2872  }
2873  }
2874 
2875  // If there was an error communicating with the partner, mark the
2876  // partner as unavailable.
2877  if (!error_message.empty()) {
2878  communication_state_->setPartnerState("unavailable");
2879  }
2880 
2881  // Invoke post request action if it was specified.
2882  if (post_request_action) {
2883  post_request_action(error_message.empty(),
2884  error_message,
2885  rcode);
2886  }
2887  },
2889  std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
2890  std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
2891  std::bind(&HAService::clientCloseHandler, this, ph::_1)
2892  );
2893 }
2894 
2897  if (getCurrState() == HA_PARTNER_DOWN_ST) {
2898  sync_complete_notified_ = true;
2899  } else {
2901  }
2903  "Server successfully notified about the synchronization completion."));
2904 }
2905 
2907 HAService::verifyAsyncResponse(const HttpResponsePtr& response, int& rcode) {
2908  // Set the return code to error in case of early throw.
2909  rcode = CONTROL_RESULT_ERROR;
2910  // The response must cast to JSON type.
2911  HttpResponseJsonPtr json_response =
2912  boost::dynamic_pointer_cast<HttpResponseJson>(response);
2913  if (!json_response) {
2914  isc_throw(CtrlChannelError, "no valid HTTP response found");
2915  }
2916 
2917  // Body holds the response to our command.
2918  ConstElementPtr body = json_response->getBodyAsJson();
2919  if (!body) {
2920  isc_throw(CtrlChannelError, "no body found in the response");
2921  }
2922 
2923  // Body should contain a list of responses from multiple servers.
2924  if (body->getType() != Element::list) {
2925  // Some control agent errors are returned as a map.
2926  if (body->getType() == Element::map) {
2927  ElementPtr list = Element::createList();
2928  ElementPtr answer = Element::createMap();
2929  answer->set(CONTROL_RESULT, Element::create(rcode));
2930  ConstElementPtr text = body->get(CONTROL_TEXT);
2931  if (text) {
2932  answer->set(CONTROL_TEXT, text);
2933  }
2934  list->add(answer);
2935  body = list;
2936  } else {
2937  isc_throw(CtrlChannelError, "body of the response must be a list");
2938  }
2939  }
2940 
2941  // There must be at least one response.
2942  if (body->empty()) {
2943  isc_throw(CtrlChannelError, "list of responses must not be empty");
2944  }
2945 
2946  // Check if the status code of the first response. We don't support multiple
2947  // at this time, because we always send a request to a single location.
2948  ConstElementPtr args = parseAnswer(rcode, body->get(0));
2949  if ((rcode != CONTROL_RESULT_SUCCESS) &&
2950  (rcode != CONTROL_RESULT_EMPTY)) {
2951  std::ostringstream s;
2952  // Include an error text if available.
2953  if (args && args->getType() == Element::string) {
2954  s << args->stringValue() << ", ";
2955  }
2956  // Include an error code.
2957  s << "error code " << rcode;
2958 
2959  if (rcode == CONTROL_RESULT_COMMAND_UNSUPPORTED) {
2960  isc_throw(CommandUnsupportedError, s.str());
2961  } else {
2962  isc_throw(CtrlChannelError, s.str());
2963  }
2964  }
2965 
2966  return (args);
2967 }
2968 
2969 bool
2970 HAService::clientConnectHandler(const boost::system::error_code& ec, int tcp_native_fd) {
2971 
2972  // If client is running it's own IOService we do NOT want to
2973  // register the socket with IfaceMgr.
2974  if (client_->getThreadIOService()) {
2975  return (true);
2976  }
2977 
2978  // If things look OK register the socket with Interface Manager. Note
2979  // we don't register if the FD is < 0 to avoid an exception throw.
2980  // It is unlikely that this will occur but we want to be liberal
2981  // and avoid issues.
2982  if ((!ec || (ec.value() == boost::asio::error::in_progress))
2983  && (tcp_native_fd >= 0)) {
2984  // External socket callback is a NOP. Ready events handlers are
2985  // run by an explicit call IOService ready in kea-dhcp<n> code.
2986  // We are registering the socket only to interrupt main-thread
2987  // select().
2988  IfaceMgr::instance().addExternalSocket(tcp_native_fd,
2989  std::bind(&HAService::socketReadyHandler, this, ph::_1)
2990  );
2991  }
2992 
2993  // If ec.value() == boost::asio::error::already_connected, we should already
2994  // be registered, so nothing to do. If it is any other value, then connect
2995  // failed and Connection logic should handle that, not us, so no matter
2996  // what happens we're returning true.
2997  return (true);
2998 }
2999 
3000 void
3001 HAService::socketReadyHandler(int tcp_native_fd) {
3002  // If the socket is ready but does not belong to one of our client's
3003  // ongoing transactions, we close it. This will unregister it from
3004  // IfaceMgr and ensure the client starts over with a fresh connection
3005  // if it needs to do so.
3006  client_->closeIfOutOfBand(tcp_native_fd);
3007 }
3008 
3009 void
3010 HAService::clientCloseHandler(int tcp_native_fd) {
3011  if (tcp_native_fd >= 0) {
3012  IfaceMgr::instance().deleteExternalSocket(tcp_native_fd);
3013  }
3014 };
3015 
3016 size_t
3018  if (MultiThreadingMgr::instance().getMode()) {
3019  std::lock_guard<std::mutex> lock(mutex_);
3020  return (pending_requests_.size());
3021  } else {
3022  return (pending_requests_.size());
3023  }
3024 }
3025 
3026 template<typename QueryPtrType>
3027 int
3028 HAService::getPendingRequest(const QueryPtrType& query) {
3029  if (MultiThreadingMgr::instance().getMode()) {
3030  std::lock_guard<std::mutex> lock(mutex_);
3031  return (getPendingRequestInternal(query));
3032  } else {
3033  return (getPendingRequestInternal(query));
3034  }
3035 }
3036 
3037 template<typename QueryPtrType>
3038 int
3039 HAService::getPendingRequestInternal(const QueryPtrType& query) {
3040  if (pending_requests_.count(query) == 0) {
3041  return (0);
3042  } else {
3043  return (pending_requests_[query]);
3044  }
3045 }
3046 
3047 void
3049  // Since this function is used as CS callback all exceptions must be
3050  // suppressed (except the @ref MultiThreadingInvalidOperation), unlikely
3051  // though they may be.
3052  // The @ref MultiThreadingInvalidOperation is propagated to the scope of the
3053  // @ref MultiThreadingCriticalSection constructor.
3054  try {
3055  if (client_) {
3056  client_->checkPermissions();
3057  }
3058 
3059  if (listener_) {
3060  listener_->checkPermissions();
3061  }
3062  } catch (const isc::MultiThreadingInvalidOperation& ex) {
3064  .arg(ex.what());
3065  // The exception needs to be propagated to the caller of the
3066  // @ref MultiThreadingCriticalSection constructor.
3067  throw;
3068  } catch (const std::exception& ex) {
3070  .arg(ex.what());
3071  }
3072 }
3073 
3074 void
3076  // Add critical section callbacks.
3077  MultiThreadingMgr::instance().addCriticalSectionCallbacks("HA_MT",
3079  std::bind(&HAService::pauseClientAndListener, this),
3080  std::bind(&HAService::resumeClientAndListener, this));
3081 
3082  if (client_) {
3083  client_->start();
3084  }
3085 
3086  if (listener_) {
3087  listener_->start();
3088  }
3089 }
3090 
3091 void
3093  // Since this function is used as CS callback all exceptions must be
3094  // suppressed, unlikely though they may be.
3095  try {
3096  if (client_) {
3097  client_->pause();
3098  }
3099 
3100  if (listener_) {
3101  listener_->pause();
3102  }
3103  } catch (const std::exception& ex) {
3105  .arg(ex.what());
3106  }
3107 }
3108 
3109 void
3111  // Since this function is used as CS callback all exceptions must be
3112  // suppressed, unlikely though they may be.
3113  try {
3114  if (client_) {
3115  client_->resume();
3116  }
3117 
3118  if (listener_) {
3119  listener_->resume();
3120  }
3121  } catch (std::exception& ex) {
3123  .arg(ex.what());
3124  }
3125 }
3126 
3127 void
3129  // Remove critical section callbacks.
3130  MultiThreadingMgr::instance().removeCriticalSectionCallbacks("HA_MT");
3131 
3132  if (client_) {
3133  client_->stop();
3134  }
3135 
3136  if (listener_) {
3137  listener_->stop();
3138  }
3139 }
3140 
3141 // Explicit instantiations.
3142 template int HAService::getPendingRequest(const Pkt4Ptr&);
3143 template int HAService::getPendingRequest(const Pkt6Ptr&);
3144 
3145 } // end of namespace isc::ha
3146 } // end of namespace isc
void defineState(unsigned int value, const std::string &label, StateHandler handler, const StatePausing &state_pausing=STATE_PAUSE_NEVER)
Adds an state value and associated label to the set of states.
Definition: state_model.cc:196
config::CmdHttpListenerPtr listener_
HTTP listener instance used to receive and respond to HA commands and lease updates.
Definition: ha_service.h:1163
Exception thrown when a worker thread is trying to stop or pause the respective thread pool (which wo...
static const int NOP_EVT
Signifies that no event has occurred.
Definition: state_model.h:292
static data::ConstElementPtr createHAReset(const HAServerType &server_type)
Creates ha-reset command.
const isc::log::MessageID HA_SYNC_FAILED
Definition: ha_messages.h:103
const int HA_TERMINATED_ST
HA service terminated state.
static const int HA_CONTROL_RESULT_MAINTENANCE_NOT_ALLOWED
Control result returned in response to ha-maintenance-notify.
Definition: ha_service.h:72
#define LOG_WARN(LOGGER, MESSAGE)
Macro to conveniently test warn output and log it.
Definition: macros.h:26
virtual void defineStates()
Defines states of the HA service.
Definition: ha_service.cc:172
Represents HTTP Host header.
Definition: http_header.h:68
const isc::log::MessageID HA_MAINTENANCE_NOTIFY_COMMUNICATIONS_FAILED
Definition: ha_messages.h:80
void serveDefaultScopes()
Instructs the HA service to serve default scopes.
Definition: ha_service.cc:1003
data::ConstElementPtr processHAReset()
Processes ha-reset command and returns a response.
Definition: ha_service.cc:1625
const int HA_COMMUNICATION_RECOVERY_ST
Communication recovery state.
void pauseClientAndListener()
Pauses client and(or) listener thread pool operations.
Definition: ha_service.cc:3092
const isc::log::MessageID HA_STATE_TRANSITION
Definition: ha_messages.h:98
const int HA_HOT_STANDBY_ST
Hot standby state.
const char * CONTROL_RESULT
String used for result, i.e. integer status ("result")
Structure that holds a lease for IPv4 address.
Definition: lease.h:289
bool leaseUpdateComplete(QueryPtrType &query, const hooks::ParkingLotHandlePtr &parking_lot)
Handle last pending request for this query.
Definition: ha_service.cc:1275
const isc::log::MessageID HA_LEASES_BACKLOG_COMMUNICATIONS_FAILED
Definition: ha_messages.h:56
static const int HA_MAINTENANCE_CANCEL_EVT
ha-maintenance-cancel command received.
Definition: ha_service.h:65
void defineEvent(unsigned int value, const std::string &label)
Adds an event value and associated label to the set of events.
Definition: state_model.cc:170
const int DBGLVL_TRACE_BASIC
Trace basic operations.
Definition: log_dbglevels.h:69
bool doOnExit()
Checks if on exit flag is true.
Definition: state_model.cc:347
void readyStateHandler()
Handler for "ready" state.
Definition: ha_service.cc:604
void serveNoScopes()
Disables all scopes.
#define LOG_INFO(LOGGER, MESSAGE)
Macro to conveniently test info output and log it.
Definition: macros.h:20
ConstElementPtr createAnswer(const int status_code, const std::string &text, const ConstElementPtr &arg)
const StatePtr getState(unsigned int value)
Fetches the state referred to by value.
Definition: state_model.cc:213
const int CONTROL_RESULT_SUCCESS
Status code indicating a successful operation.
void asyncSendRequest(const Url &url, const asiolink::TlsContextPtr &tls_context, const HttpRequestPtr &request, const HttpResponsePtr &response, const RequestHandler &request_callback, const RequestTimeout &request_timeout=RequestTimeout(10000), const ConnectHandler &connect_callback=ConnectHandler(), const HandshakeHandler &handshake_callback=HandshakeHandler(), const CloseHandler &close_callback=CloseHandler())
Queues new asynchronous HTTP request for a given URL.
Definition: client.cc:1966
size_t pendingRequestSize()
Get the number of entries in the pending request map.
Definition: ha_service.cc:3017
data::ConstElementPtr processScopes(const std::vector< std::string > &scopes)
Processes ha-scopes command and returns a response.
Definition: ha_service.cc:2532
void scheduleHeartbeat()
Schedules asynchronous heartbeat to a peer if it is not scheduled.
Definition: ha_service.cc:1788
std::map< std::string, PeerConfigPtr > PeerConfigMap
Map of the servers&#39; configurations.
Definition: ha_config.h:232
void asyncEnableDHCPService(http::HttpClient &http_client, const std::string &server_name, PostRequestCallback post_request_action)
Schedules asynchronous "dhcp-enable" command to the specified server.
Definition: ha_service.cc:1884
boost::shared_ptr< HttpResponseJson > HttpResponseJsonPtr
Pointer to the HttpResponseJson object.
Definition: response_json.h:24
const isc::log::MessageID HA_INVALID_PARTNER_STATE_COMMUNICATION_RECOVERY
Definition: ha_messages.h:49
const isc::log::MessageID HA_LEASE_UPDATES_ENABLED
Definition: ha_messages.h:68
bool unpause()
Unpauses the HA state machine with logging.
Definition: ha_service.cc:982
const isc::log::MessageID HA_LEASES_BACKLOG_NOTHING_TO_SEND
Definition: ha_messages.h:58
void passiveBackupStateHandler()
Handler for "passive-backup" state.
Definition: ha_service.cc:586
const int HA_PARTNER_DOWN_ST
Partner down state.
void asyncSyncLeases()
Asynchronously reads leases from a peer and updates local lease database.
Definition: ha_service.cc:1971
const isc::log::MessageID HA_LEASE_UPDATE_DELETE_FAILED_ON_PEER
Definition: ha_messages.h:71
void asyncSyncCompleteNotify(http::HttpClient &http_client, const std::string &server_name, PostRequestCallback post_request_action)
Schedules asynchronous "ha-sync-complete-notify" command to the specified server. ...
Definition: ha_service.cc:2815
data::ConstElementPtr processHeartbeat()
Processes ha-heartbeat command and returns a response.
Definition: ha_service.cc:1602
QueryFilter query_filter_
Selects queries to be processed/dropped.
Definition: ha_service.h:1169
An abstract API for lease database.
LeaseUpdateBacklog lease_update_backlog_
Backlog of DHCP lease updates.
Definition: ha_service.h:1283
OpType
Type of the lease update (operation type).
const isc::log::MessageID HA_MAINTENANCE_STARTED_IN_PARTNER_DOWN
Definition: ha_messages.h:85
const isc::log::MessageID HA_LEASES_SYNC_COMMUNICATIONS_FAILED
Definition: ha_messages.h:61
CommunicationStatePtr communication_state_
Holds communication state with a peer.
Definition: ha_service.h:1166
static std::unordered_set< std::string > ha_commands6_
List of commands used by the High Availability in v6.
virtual void verifyEvents()
Verifies events used by the HA service.
Definition: ha_service.cc:158
const int HA_IN_MAINTENANCE_ST
In maintenance state.
HTTP request/response timeout value.
Definition: client.h:90
static const int HA_SYNCING_SUCCEEDED_EVT
Lease database synchronization succeeded.
Definition: ha_service.h:56
bool clientConnectHandler(const boost::system::error_code &ec, int tcp_native_fd)
HttpClient connect callback handler.
Definition: ha_service.cc:2970
const int HA_LOAD_BALANCING_ST
Load balancing state.
HAServerType server_type_
DHCP server type.
Definition: ha_service.h:1156
#define LOG_ERROR(LOGGER, MESSAGE)
Macro to conveniently test error output and log it.
Definition: macros.h:32
const int CONTROL_RESULT_ERROR
Status code indicating a general failure.
boost::shared_ptr< Element > ElementPtr
Definition: data.h:24
const isc::log::MessageID HA_LEASE_SYNC_STALE_LEASE4_SKIP
Definition: ha_messages.h:65
static const int HA_MAINTENANCE_NOTIFY_EVT
ha-maintenance-notify command received.
Definition: ha_service.h:59
const isc::log::MessageID HA_CONFIG_LEASE_UPDATES_DISABLED_REMINDER
Definition: ha_messages.h:33
const isc::log::MessageID HA_MAINTENANCE_NOTIFY_FAILED
Definition: ha_messages.h:81
const isc::log::MessageID HA_RESET_FAILED
Definition: ha_messages.h:91
data::ConstElementPtr processMaintenanceCancel()
Processes ha-maintenance-cancel command and returns a response.
Definition: ha_service.cc:2719
const isc::log::MessageID HA_COMMUNICATION_INTERRUPTED
Definition: ha_messages.h:20
static data::ConstElementPtr createLease6GetPage(const dhcp::Lease6Ptr &lease6, const uint32_t limit)
Creates lease6-get-page command.
void waitingStateHandler()
Handler for "waiting" state.
Definition: ha_service.cc:793
const isc::log::MessageID HA_LEASES_BACKLOG_SUCCESS
Definition: ha_messages.h:60
void updatePendingRequest(QueryPtrType &query)
Update pending request counter for this query.
Definition: ha_service.cc:1308
const int HA_PASSIVE_BACKUP_ST
In passive-backup state with a single active server and backup servers.
void partnerInMaintenanceStateHandler()
Handler for "partner-in-maintenance" state.
Definition: ha_service.cc:547
HAServerType
Lists possible server types for which HA service is created.
const int HA_BACKUP_ST
Backup state.
virtual ~HAService()
Destructor.
Definition: ha_service.cc:136
const isc::log::MessageID HA_LEASE_UPDATES_DISABLED
Definition: ha_messages.h:67
const isc::log::MessageID HA_RESET_COMMUNICATIONS_FAILED
Definition: ha_messages.h:90
bool shouldTerminate() const
Indicates if the server should transition to the terminated state as a result of high clock skew...
Definition: ha_service.cc:1097
void logFailedLeaseUpdates(const dhcp::PktPtr &query, const data::ConstElementPtr &args) const
Log failed lease updates.
Definition: ha_service.cc:1501
boost::shared_ptr< Lease > LeasePtr
Pointer to the lease object.
Definition: lease.h:21
void serveDefaultScopes()
Serve default scopes for the given HA mode.
virtual const char * what() const
Returns a C-style character string of the cause of the exception.
const isc::log::MessageID HA_PAUSE_CLIENT_LISTENER_ILLEGAL
Definition: ha_messages.h:89
dhcp::NetworkStatePtr network_state_
Pointer to the state of the DHCP service (enabled/disabled).
Definition: ha_service.h:1150
static const int HA_MAINTENANCE_START_EVT
ha-maintenance-start command received.
Definition: ha_service.h:62
unsigned int getLastEvent() const
Fetches the model&#39;s last event.
Definition: state_model.cc:367
const isc::log::MessageID HA_RESUME_CLIENT_LISTENER_FAILED
Definition: ha_messages.h:93
const isc::log::MessageID HA_SYNC_COMPLETE_NOTIFY_FAILED
Definition: ha_messages.h:101
virtual void runModel(unsigned int event)
Processes events through the state model.
Definition: state_model.cc:112
const int CONTROL_RESULT_EMPTY
Status code indicating that the specified command was completed correctly, but failed to produce any ...
asiolink::IOServicePtr io_service_
Pointer to the IO service object shared between this hooks library and the DHCP server.
Definition: ha_service.h:1147
const int HA_WAITING_ST
Server waiting state, i.e. waiting for another server to be ready.
const isc::log::MessageID HA_INVALID_PARTNER_STATE_HOT_STANDBY
Definition: ha_messages.h:50
data::ConstElementPtr processContinue()
Processes ha-continue command and returns a response.
Definition: ha_service.cc:2545
#define isc_throw(type, stream)
A shortcut macro to insert known values into exception arguments.
void stop()
Stops the stopwatch.
Definition: stopwatch.cc:35
Holds communication state between DHCPv4 servers.
void resumeClientAndListener()
Resumes client and(or) listener thread pool operations.
Definition: ha_service.cc:3110
boost::shared_ptr< Pkt6 > Pkt6Ptr
A pointer to Pkt6 packet.
Definition: pkt6.h:28
void checkPermissionsClientAndListener()
Check client and(or) listener current thread permissions to perform thread pool state transition...
Definition: ha_service.cc:3048
data::ConstElementPtr processSyncCompleteNotify()
Process ha-sync-complete-notify command and returns a response.
Definition: ha_service.cc:2896
Definition: edns.h:19
const int HA_PARTNER_IN_MAINTENANCE_ST
Partner in-maintenance state.
boost::shared_ptr< PostHttpRequestJson > PostHttpRequestJsonPtr
Pointer to PostHttpRequestJson.
const int HA_READY_ST
Server ready state, i.e. synchronized database, can enable DHCP service.
static const int HA_HEARTBEAT_COMPLETE_EVT
Finished heartbeat command.
Definition: ha_service.h:47
const isc::log::MessageID HA_LEASE_UPDATE_CREATE_UPDATE_FAILED_ON_PEER
Definition: ha_messages.h:70
const isc::log::MessageID HA_DHCP_DISABLE_COMMUNICATIONS_FAILED
Definition: ha_messages.h:39
static const int HA_LEASE_UPDATES_COMPLETE_EVT
Finished lease updates commands.
Definition: ha_service.h:50
boost::shared_ptr< Lease4 > Lease4Ptr
Pointer to a Lease4 structure.
Definition: lease.h:278
static data::ConstElementPtr createLease4Update(const dhcp::Lease4 &lease4)
Creates lease4-update command.
void partnerDownStateHandler()
Handler for "partner-down" state.
Definition: ha_service.cc:457
void syncingStateHandler()
Handler for "syncing" state.
Definition: ha_service.cc:683
static const int HA_SYNCED_PARTNER_UNAVAILABLE_EVT
The heartbeat command failed after receiving ha-sync-complete-notify command from the partner...
Definition: ha_service.h:69
void clear()
Removes all lease updates from the queue.
boost::shared_ptr< HttpResponse > HttpResponsePtr
Pointer to the HttpResponse object.
Definition: response.h:78
Utility class to measure code execution times.
Definition: stopwatch.h:35
void asyncSendHeartbeat()
Starts asynchronous heartbeat to a peer.
Definition: ha_service.cc:1635
std::set< std::string > getServedScopes() const
Returns served scopes.
void serveScopes(const std::vector< std::string > &scopes)
Enables selected scopes.
const isc::log::MessageID HA_LEASES_BACKLOG_START
Definition: ha_messages.h:59
boost::shared_ptr< Lease4Collection > Lease4CollectionPtr
A shared pointer to the collection of IPv4 leases.
Definition: lease.h:485
void inMaintenanceStateHandler()
Handler for the "in-maintenance" state.
Definition: ha_service.cc:432
A generic exception that is thrown when an unexpected error condition occurs.
void asyncSendLeaseUpdate(const QueryPtrType &query, const HAConfig::PeerConfigPtr &config, const data::ConstElementPtr &command, const hooks::ParkingLotHandlePtr &parking_lot)
Asynchronously sends lease update to the peer.
Definition: ha_service.cc:1329
bool doOnEntry()
Checks if on entry flag is true.
Definition: state_model.cc:339
const char * CONTROL_TEXT
String used for storing textual description ("text")
static data::ConstElementPtr createSyncCompleteNotify(const HAServerType &server_type)
Creates ha-sync-complete-notify command.
const isc::log::MessageID HA_LEASE_SYNC_FAILED
Definition: ha_messages.h:64
bool push(const OpType op_type, const dhcp::LeasePtr &lease)
Appends lease update to the queue.
const isc::log::MessageID HA_SERVICE_STARTED
Definition: ha_messages.h:95
std::string stateToString(int state)
Returns state name.
std::string rfc1123Format() const
Returns time value formatted as specified in RFC 1123.
Definition: date_time.cc:30
const isc::log::MessageID HA_LOCAL_DHCP_ENABLE
Definition: ha_messages.h:76
boost::shared_ptr< Pkt4 > Pkt4Ptr
A pointer to Pkt4 object.
Definition: pkt4.h:544
boost::shared_ptr< const Element > ConstElementPtr
Definition: data.h:27
static std::unordered_set< std::string > ha_commands4_
List of commands used by the High Availability in v4.
std::string logFormatLastDuration() const
Returns the last measured duration in the format directly usable in log messages. ...
Definition: stopwatch.cc:75
const isc::log::MessageID HA_STATE_TRANSITION_PASSIVE_BACKUP
Definition: ha_messages.h:99
const isc::log::MessageID HA_HEARTBEAT_FAILED
Definition: ha_messages.h:44
data::ConstElementPtr processMaintenanceNotify(const bool cancel)
Processes ha-maintenance-notify command and returns a response.
Definition: ha_service.cc:2553
const isc::log::MessageID HA_DHCP_DISABLE_FAILED
Definition: ha_messages.h:40
size_t size()
Returns the current size of the queue.
unsigned int getNextEvent() const
Fetches the model&#39;s next event.
Definition: state_model.cc:373
bool inScope(dhcp::Pkt4Ptr &query4)
Checks if the DHCPv4 query should be processed by this server.
Definition: ha_service.cc:1008
void terminatedStateHandler()
Handler for "terminated" state.
Definition: ha_service.cc:772
const isc::log::MessageID HA_SYNC_COMPLETE_NOTIFY_COMMUNICATIONS_FAILED
Definition: ha_messages.h:100
bool wasOverflown()
Checks if the queue was overflown.
void stopClientAndListener()
Stop the client and(or) listener instances.
Definition: ha_service.cc:3128
const isc::log::MessageID HA_LEASE_UPDATE_COMMUNICATIONS_FAILED
Definition: ha_messages.h:69
const isc::log::MessageID HA_LEASE_UPDATE_FAILED
Definition: ha_messages.h:72
void serveFailoverScopes()
Enable scopes required in failover case.
void localDisableDHCPService()
Disables local DHCP service.
Definition: ha_service.cc:1961
boost::shared_ptr< isc::dhcp::Pkt > PktPtr
A pointer to either Pkt4 or Pkt6 packet.
Definition: pkt.h:797
unsigned int getCurrState() const
Fetches the model&#39;s current state.
Definition: state_model.cc:355
std::function< void(const bool, const std::string &, const int)> PostRequestCallback
Callback invoked when request was sent and a response received or an error occurred.
Definition: ha_service.h:82
void adjustNetworkState()
Enables or disables network state depending on the served scopes.
Definition: ha_service.cc:1038
const isc::log::MessageID HA_INVALID_PARTNER_STATE_LOAD_BALANCING
Definition: ha_messages.h:51
This class parses and generates time values used in HTTP.
Definition: date_time.h:41
const isc::log::MessageID HA_SYNC_START
Definition: ha_messages.h:105
static data::ConstElementPtr createLease4Delete(const dhcp::Lease4 &lease4)
Creates lease4-del command.
void startModel(const int start_state)
Begins execution of the model.
Definition: state_model.cc:100
void unpauseModel()
Unpauses state model.
Definition: state_model.cc:276
Represents HTTP response with JSON content.
Definition: response_json.h:34
void asyncSyncLeasesInternal(http::HttpClient &http_client, const std::string &server_name, const unsigned int max_period, const dhcp::LeasePtr &last_lease, PostSyncCallback post_sync_action, const bool dhcp_disabled)
Implements fetching one page of leases during synchronization.
Definition: ha_service.cc:2018
std::function< void(const bool, const std::string &, const bool)> PostSyncCallback
Callback invoked when lease database synchronization is complete.
Definition: ha_service.h:91
HTTP client class.
Definition: client.h:87
void communicationRecoveryHandler()
Handler for the "communication-recovery" state.
Definition: ha_service.cc:240
A standard control channel exception that is thrown if a function is there is a problem with one of t...
ConstElementPtr parseAnswer(int &rcode, const ConstElementPtr &msg)
Defines the logger used by the top-level component of kea-lfc.
const isc::log::MessageID HA_STATE_MACHINE_PAUSED
Definition: ha_messages.h:97
bool shouldPartnerDown() const
Indicates if the server should transition to the partner down state.
Definition: ha_service.cc:1070
const isc::log::MessageID HA_LOCAL_DHCP_DISABLE
Definition: ha_messages.h:75
const isc::log::MessageID HA_MAINTENANCE_NOTIFY_CANCEL_COMMUNICATIONS_FAILED
Definition: ha_messages.h:78
data::ConstElementPtr processMaintenanceStart()
Processes ha-maintenance-start command and returns a response.
Definition: ha_service.cc:2589
bool isMaintenanceCanceled() const
Convenience method checking if the current state is a result of canceling the maintenance.
Definition: ha_service.cc:1110
HAConfigPtr config_
Pointer to the HA hooks library configuration.
Definition: ha_service.h:1153
void startHeartbeat()
Unconditionally starts one heartbeat to a peer.
Definition: ha_service.cc:1795
const isc::log::MessageID HA_MAINTENANCE_STARTED
Definition: ha_messages.h:84
data::ConstElementPtr processStatusGet() const
Processes status-get command and returns a response.
Definition: ha_service.cc:1553
const isc::log::MessageID HA_HEARTBEAT_COMMUNICATIONS_FAILED
Definition: ha_messages.h:43
void transition(unsigned int state, unsigned int event)
Sets up the model to transition into given state with a given event.
Definition: state_model.cc:264
bool sync_complete_notified_
An indicator that a partner sent ha-sync-complete-notify command.
Definition: ha_service.h:1292
void startClientAndListener()
Start the client and(or) listener instances.
Definition: ha_service.cc:3075
http::HttpClientPtr client_
HTTP client instance used to send HA commands and lease updates.
Definition: ha_service.h:1159
const isc::log::MessageID HA_DHCP_ENABLE_FAILED
Definition: ha_messages.h:42
void normalStateHandler()
Handler for the "hot-standby" and "load-balancing" states.
Definition: ha_service.cc:357
This file contains several functions and constants that are used for handling commands and responses ...
bool sendLeaseUpdatesFromBacklog()
Attempts to send all lease updates from the backlog synchronously.
Definition: ha_service.cc:2422
static std::string HAModeToString(const HAMode &ha_mode)
Returns HA mode name.
Definition: ha_config.cc:225
void localEnableDHCPService()
Enables local DHCP service.
Definition: ha_service.cc:1966
a common structure for IPv4 and IPv6 leases
Definition: lease.h:30
virtual void defineEvents()
Defines events used by the HA service.
Definition: ha_service.cc:144
std::string getStateLabel(const int state) const
Fetches the label associated with an state value.
Definition: state_model.cc:421
boost::shared_ptr< Lease6Collection > Lease6CollectionPtr
A shared pointer to the collection of IPv6 leases.
Definition: lease.h:639
boost::shared_ptr< NetworkState > NetworkStatePtr
Pointer to the NetworkState object.
unsigned int getPrevState() const
Fetches the model&#39;s previous state.
Definition: state_model.cc:361
#define LOG_DEBUG(LOGGER, LEVEL, MESSAGE)
Macro to conveniently test debug output and log it.
Definition: macros.h:14
static data::ConstElementPtr createLease6BulkApply(const dhcp::Lease6CollectionPtr &leases, const dhcp::Lease6CollectionPtr &deleted_leases)
Creates lease6-bulk-apply command.
isc::log::Logger ha_logger("ha-hooks")
Definition: ha_log.h:17
bool isPartnerStateInvalid() const
Indicates if the partner&#39;s state is invalid.
Definition: ha_service.cc:1115
bool shouldSendLeaseUpdates(const HAConfig::PeerConfigPtr &peer_config) const
Checks if the lease updates should be sent as result of leases allocation or release.
Definition: ha_service.cc:1456
const isc::log::MessageID HA_LEASE_SYNC_STALE_LEASE6_SKIP
Definition: ha_messages.h:66
void backupStateHandler()
Handler for the "backup" state.
Definition: ha_service.cc:225
int getPendingRequest(const QueryPtrType &query)
Get the number of scheduled requests for a given query.
Definition: ha_service.cc:3028
const isc::log::MessageID HA_SYNC_SUCCESSFUL
Definition: ha_messages.h:106
const isc::log::MessageID HA_LEASES_BACKLOG_FAILED
Definition: ha_messages.h:57
static const int HA_SYNCING_FAILED_EVT
Lease database synchronization failed.
Definition: ha_service.h:53
bool inScope(const dhcp::Pkt4Ptr &query4, std::string &scope_class) const
Checks if this server should process the DHCPv4 query.
A multi-threaded HTTP listener that can process API commands requests.
Role
Server&#39;s role in the High Availability setup.
Definition: ha_config.h:70
const isc::log::MessageID HA_MAINTENANCE_SHUTDOWN_SAFE
Definition: ha_messages.h:83
size_t asyncSendLeaseUpdates(const dhcp::Pkt4Ptr &query, const dhcp::Lease4CollectionPtr &leases, const dhcp::Lease4CollectionPtr &deleted_leases, const hooks::ParkingLotHandlePtr &parking_lot)
Schedules asynchronous IPv4 leases updates.
Definition: ha_service.cc:1145
boost::shared_ptr< ParkingLotHandle > ParkingLotHandlePtr
Pointer to the parking lot handle.
Definition: parking_lots.h:381
void asyncSendHAReset(http::HttpClient &http_client, const HAConfig::PeerConfigPtr &remote_config, PostRequestCallback post_request_action)
Sends ha-reset command to partner asynchronously.
Definition: ha_service.cc:2463
dhcp::LeasePtr pop(OpType &op_type)
Returns the next lease update and removes it from the queue.
const isc::log::MessageID HA_PAUSE_CLIENT_LISTENER_FAILED
Definition: ha_messages.h:88
const isc::log::MessageID HA_CONFIG_LEASE_SYNCING_DISABLED_REMINDER
Definition: ha_messages.h:30
const isc::log::MessageID HA_MAINTENANCE_NOTIFY_CANCEL_FAILED
Definition: ha_messages.h:79
static std::string roleToString(const HAConfig::PeerConfig::Role &role)
Returns role name.
Definition: ha_config.cc:79
const isc::log::MessageID HA_STATE_MACHINE_CONTINUED
Definition: ha_messages.h:96
void socketReadyHandler(int tcp_native_fd)
IfaceMgr external socket ready callback handler.
Definition: ha_service.cc:3001
static data::ConstElementPtr createHeartbeat(const HAServerType &server_type)
Creates ha-heartbeat command for DHCP server.
std::string ClientClass
Defines a single class name.
Definition: classify.h:40
int getNormalState() const
Returns normal operation state for the current configuration.
Definition: ha_service.cc:966
const EventPtr & getEvent(unsigned int value)
Fetches the event referred to by value.
Definition: state_model.cc:186
void conditionalLogPausedState() const
Logs if the server is paused in the current state.
Definition: ha_service.cc:992
const isc::log::MessageID HA_TERMINATED_RESTART_PARTNER
Definition: ha_messages.h:108
constexpr long TIMEOUT_DEFAULT_HTTP_CLIENT_REQUEST
Timeout for the HTTP clients awaiting a response to a request.
Definition: timeouts.h:38
data::ConstElementPtr processSynchronize(const std::string &server_name, const unsigned int max_period)
Processes ha-sync command and returns a response.
Definition: ha_service.cc:2214
bool isModelPaused() const
Returns whether or not the model is paused.
Definition: state_model.cc:415
const int HA_SYNCING_ST
Synchronizing database state.
const isc::log::MessageID HA_DHCP_ENABLE_COMMUNICATIONS_FAILED
Definition: ha_messages.h:41
const isc::log::MessageID HA_LEASES_SYNC_LEASE_PAGE_RECEIVED
Definition: ha_messages.h:63
boost::shared_ptr< Lease6 > Lease6Ptr
Pointer to a Lease6 structure.
Definition: lease.h:487
const isc::log::MessageID HA_LEASES_SYNC_FAILED
Definition: ha_messages.h:62
static data::ConstElementPtr createMaintenanceNotify(const bool cancel, const HAServerType &server_type)
Creates ha-maintenance-notify command.
data::ConstElementPtr verifyAsyncResponse(const http::HttpResponsePtr &response, int &rcode)
Checks if the response is valid or contains an error.
Definition: ha_service.cc:2907
static data::ConstElementPtr createDHCPEnable(const HAServerType &server_type)
Creates dhcp-enable command for DHCP server.
const char * MessageID
Definition: message_types.h:15
void clientCloseHandler(int tcp_native_fd)
HttpClient close callback handler.
Definition: ha_service.cc:3010
void verboseTransition(const unsigned state)
Transitions to a desired state and logs it.
Definition: ha_service.cc:901
bool shouldQueueLeaseUpdates(const HAConfig::PeerConfigPtr &peer_config) const
Checks if the lease updates should be queued.
Definition: ha_service.cc:1488
boost::shared_ptr< HAConfig > HAConfigPtr
Pointer to the High Availability configuration structure.
Definition: ha_config.h:786
bool sendHAReset()
Sends ha-reset command to partner synchronously.
Definition: ha_service.cc:2513
static data::ConstElementPtr createDHCPDisable(const unsigned int max_period, const HAServerType &server_type)
Creates dhcp-disable command for DHCP server.
Holds communication state between DHCPv6 servers.
const int HA_UNAVAILABLE_ST
Special state indicating that this server is unable to communicate with the partner.
void postNextEvent(unsigned int event)
Sets the next event to the given event value.
Definition: state_model.cc:320
int synchronize(std::string &status_message, const std::string &server_name, const unsigned int max_period)
Synchronizes lease database with a partner.
Definition: ha_service.cc:2222
bool clientHandshakeHandler(const boost::system::error_code &)
HttpClient handshake callback handler.
Definition: ha_service.h:1114
boost::shared_ptr< PeerConfig > PeerConfigPtr
Pointer to the server&#39;s configuration.
Definition: ha_config.h:229
const int CONTROL_RESULT_COMMAND_UNSUPPORTED
Status code indicating that the specified command is not supported.
static data::ConstElementPtr createLease4GetPage(const dhcp::Lease4Ptr &lease4, const uint32_t limit)
Creates lease4-get-page command.
const isc::log::MessageID HA_TERMINATED
Definition: ha_messages.h:107
void asyncDisableDHCPService(http::HttpClient &http_client, const std::string &server_name, const unsigned int max_period, PostRequestCallback post_request_action)
Schedules asynchronous "dhcp-disable" command to the specified server.
Definition: ha_service.cc:1804
void asyncSendLeaseUpdatesFromBacklog(http::HttpClient &http_client, const HAConfig::PeerConfigPtr &remote_config, PostRequestCallback post_request_action)
Sends lease updates from backlog to partner asynchronously.
Definition: ha_service.cc:2346