diff --git a/CMakeLists.txt b/CMakeLists.txt index 186e5b670..674c1c158 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,7 +30,6 @@ set(PlatformLib platform) include_directories(${CMAKE_SOURCE_DIR}/include) include_directories(${CMAKE_SOURCE_DIR}/include/core) include_directories(${CMAKE_SOURCE_DIR}/include/core/federated) -include_directories(${CMAKE_SOURCE_DIR}/include/core/federated/network) include_directories(${CMAKE_SOURCE_DIR}/include/core/modal_models) include_directories(${CMAKE_SOURCE_DIR}/include/core/platform) include_directories(${CMAKE_SOURCE_DIR}/include/core/threaded) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index c8b487dfd..7cc33efc2 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -18,7 +18,6 @@ list(APPEND REACTORC_SOURCES ${GENERAL_SOURCES}) # Add sources for either threaded or single-threaded runtime if(DEFINED FEDERATED) include(federated/CMakeLists.txt) - include(federated/network/CMakeLists.txt) endif() # Add sources for either threaded or single-threaded runtime @@ -96,10 +95,19 @@ include(${LF_ROOT}/platform/impl/CMakeLists.txt) target_link_libraries(reactor-c PUBLIC lf::platform-api) target_link_libraries(reactor-c PRIVATE lf::platform-impl) +if(DEFINED FEDERATED) + if(NOT DEFINED COMM_TYPE) + set(COMM_TYPE TCP) + endif() + include(${LF_ROOT}/network/api/CMakeLists.txt) + include(${LF_ROOT}/network/impl/CMakeLists.txt) + target_link_libraries(reactor-c PUBLIC lf::network-api) + target_link_libraries(reactor-c PRIVATE lf::network-impl) +endif() + target_include_directories(reactor-c PUBLIC ../include) target_include_directories(reactor-c PUBLIC ../include/core) target_include_directories(reactor-c PUBLIC ../include/core/federated) -target_include_directories(reactor-c PUBLIC ../include/core/federated/network) target_include_directories(reactor-c PUBLIC ../include/core/platform) target_include_directories(reactor-c PUBLIC ../include/core/modal_models) target_include_directories(reactor-c PUBLIC ../include/core/threaded) diff --git a/core/federated/RTI/CMakeLists.txt b/core/federated/RTI/CMakeLists.txt index 56217d06a..c2023813c 100644 --- a/core/federated/RTI/CMakeLists.txt +++ b/core/federated/RTI/CMakeLists.txt @@ -16,8 +16,6 @@ add_library(${RTI_LIB} STATIC ${CoreLib}/utils/util.c ${CoreLib}/tag.c ${CoreLib}/clock.c - ${CoreLib}/federated/network/net_util.c - ${CoreLib}/federated/network/socket_common.c ${CoreLib}/utils/pqueue_base.c ${CoreLib}/utils/pqueue_tag.c ${CoreLib}/utils/pqueue.c @@ -29,7 +27,6 @@ add_executable(${RTI_MAIN} main.c) target_include_directories(${RTI_LIB} PUBLIC ../../../include) target_include_directories(${RTI_LIB} PUBLIC ${IncludeDir}) target_include_directories(${RTI_LIB} PUBLIC ${IncludeDir}/federated) -target_include_directories(${RTI_LIB} PUBLIC ${IncludeDir}/federated/network) target_include_directories(${RTI_LIB} PUBLIC ${IncludeDir}/modal_models) target_include_directories(${RTI_LIB} PUBLIC ${IncludeDir}/utils) @@ -37,6 +34,10 @@ if (NOT DEFINED LOG_LEVEL) set(LOG_LEVEL 2) ENDIF(NOT DEFINED LOG_LEVEL) +if(NOT DEFINED COMM_TYPE) + set(COMM_TYPE TCP) +endif() + IF(CMAKE_BUILD_TYPE MATCHES DEBUG) # Set the LOG_LEVEL to 4 to get DEBUG messages message("-- Building RTI with DEBUG messages enabled") @@ -71,6 +72,12 @@ target_link_libraries(${RTI_LIB} PUBLIC lf::low-level-platform-impl) include(${LF_ROOT}/low_level_platform/api/CMakeLists.txt) target_link_libraries(${RTI_LIB} PUBLIC lf::low-level-platform-api) +include(${LF_ROOT}/network/impl/CMakeLists.txt) +target_link_libraries(${RTI_LIB} PUBLIC lf::network-impl) + +include(${LF_ROOT}/network/api/CMakeLists.txt) +target_link_libraries(${RTI_LIB} PUBLIC lf::network-api) + # Set the STANDALONE_RTI flag to include the rti_remote and rti_common. target_compile_definitions(${RTI_LIB} PUBLIC STANDALONE_RTI=1) @@ -78,6 +85,9 @@ target_compile_definitions(${RTI_LIB} PUBLIC STANDALONE_RTI=1) target_compile_definitions(${RTI_LIB} PUBLIC FEDERATED=1) target_compile_definitions(${RTI_LIB} PUBLIC PLATFORM_${CMAKE_SYSTEM_NAME}) +# Set communication type. +target_compile_definitions(${RTI_LIB} PUBLIC COMM_TYPE_${COMM_TYPE}) + # Set RTI Tracing target_compile_definitions(${RTI_LIB} PUBLIC RTI_TRACE) diff --git a/core/federated/RTI/main.c b/core/federated/RTI/main.c index 76e8a72cb..13e93eaa9 100644 --- a/core/federated/RTI/main.c +++ b/core/federated/RTI/main.c @@ -56,11 +56,11 @@ static void send_failed_signal(federate_info_t* fed) { if (rti.base.tracing_enabled) { tracepoint_rti_to_federate(send_FAILED, fed->enclave.id, NULL); } - int failed = write_to_socket(fed->socket, bytes_to_write, &(buffer[0])); + int failed = write_to_net(fed->net, bytes_to_write, &(buffer[0])); if (failed == 0) { LF_PRINT_LOG("RTI has sent failed signal to federate %d due to abnormal termination.", fed->enclave.id); } else { - lf_print_error("RTI failed to send failed signal to federate %d on socket ID %d.", fed->enclave.id, fed->socket); + lf_print_error("RTI failed to send failed signal to federate %d.", fed->enclave.id); } } @@ -220,6 +220,7 @@ int process_args(int argc, const char* argv[]) { rti.base.number_of_scheduling_nodes = (int32_t)num_federates; // FIXME: Loses numbers on 64-bit machines lf_print("RTI: Number of federates: %d", rti.base.number_of_scheduling_nodes); } else if (strcmp(argv[i], "-p") == 0 || strcmp(argv[i], "--port") == 0) { +#ifdef COMM_TYPE_TCP if (argc < i + 2) { lf_print_error("--port needs a short unsigned integer argument ( > 0 and < %d).", UINT16_MAX); usage(argc, argv); @@ -233,6 +234,9 @@ int process_args(int argc, const char* argv[]) { return 0; } rti.user_specified_port = (uint16_t)RTI_port; +#else + lf_print_error("--port is only available for TCP."); +#endif } else if (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "--clock_sync") == 0) { if (argc < i + 2) { lf_print_error("--clock-sync needs off|init|on."); @@ -313,9 +317,8 @@ int main(int argc, const char* argv[]) { rti.base.scheduling_nodes[i] = (scheduling_node_t*)fed_info; } - int socket_descriptor = start_rti_server(rti.user_specified_port); - if (socket_descriptor >= 0) { - wait_for_federates(socket_descriptor); + if (!start_rti_server()) { + wait_for_federates(); normal_termination = true; if (rti.base.tracing_enabled) { // No need for a mutex lock because all threads have exited. diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c index 9ed607530..d337166c1 100644 --- a/core/federated/RTI/rti_remote.c +++ b/core/federated/RTI/rti_remote.c @@ -71,9 +71,9 @@ void notify_tag_advance_grant(scheduling_node_t* e, tag_t tag) { tracepoint_rti_to_federate(send_TAG, e->id, &tag); } // This function is called in notify_advance_grant_if_safe(), which is a long - // function. During this call, the socket might close, causing the following write_to_socket + // function. During this call, the network abstraction might close, causing the following write_to_net // to fail. Consider a failure here a soft failure and update the federate's status. - if (write_to_socket(((federate_info_t*)e)->socket, message_length, buffer)) { + if (write_to_net(((federate_info_t*)e)->net, message_length, buffer)) { lf_print_error("RTI failed to send tag advance grant to federate %d.", e->id); e->state = NOT_CONNECTED; } else { @@ -104,9 +104,9 @@ void notify_provisional_tag_advance_grant(scheduling_node_t* e, tag_t tag) { tracepoint_rti_to_federate(send_PTAG, e->id, &tag); } // This function is called in notify_advance_grant_if_safe(), which is a long - // function. During this call, the socket might close, causing the following write_to_socket + // function. During this call, the network abstraction might close, causing the following write_to_net // to fail. Consider a failure here a soft failure and update the federate's status. - if (write_to_socket(((federate_info_t*)e)->socket, message_length, buffer)) { + if (write_to_net(((federate_info_t*)e)->net, message_length, buffer)) { lf_print_error("RTI failed to send tag advance grant to federate %d.", e->id); e->state = NOT_CONNECTED; } else { @@ -163,7 +163,7 @@ void notify_downstream_next_event_tag(scheduling_node_t* e, tag_t tag) { if (rti_remote->base.tracing_enabled) { tracepoint_rti_to_federate(send_DNET, e->id, &tag); } - if (write_to_socket(((federate_info_t*)e)->socket, message_length, buffer)) { + if (write_to_net(((federate_info_t*)e)->net, message_length, buffer)) { lf_print_error("RTI failed to send downstream next event tag to federate %d.", e->id); e->state = NOT_CONNECTED; } else { @@ -185,9 +185,9 @@ void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_even void handle_port_absent_message(federate_info_t* sending_federate, unsigned char* buffer) { size_t message_size = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(int64_t) + sizeof(uint32_t); - read_from_socket_fail_on_error(&sending_federate->socket, message_size, &(buffer[1]), - " RTI failed to read port absent message from federate %u.", - sending_federate->enclave.id); + read_from_net_fail_on_error(sending_federate->net, message_size, &(buffer[1]), + " RTI failed to read port absent message from federate %u.", + sending_federate->enclave.id); uint16_t reactor_port_id = extract_uint16(&(buffer[1])); uint16_t federate_id = extract_uint16(&(buffer[1 + sizeof(uint16_t)])); @@ -198,7 +198,7 @@ void handle_port_absent_message(federate_info_t* sending_federate, unsigned char } // Need to acquire the mutex lock to ensure that the thread handling - // messages coming from the socket connected to the destination does not + // messages coming from the network abstraction connected to the destination does not // issue a TAG before this message has been forwarded. LF_MUTEX_LOCK(&rti_mutex); @@ -234,8 +234,8 @@ void handle_port_absent_message(federate_info_t* sending_federate, unsigned char } // Forward the message. - write_to_socket_fail_on_error(&fed->socket, message_size + 1, buffer, &rti_mutex, - "RTI failed to forward message to federate %d.", federate_id); + write_to_net_fail_on_error(fed->net, message_size + 1, buffer, &rti_mutex, + "RTI failed to forward message to federate %d.", federate_id); LF_MUTEX_UNLOCK(&rti_mutex); } @@ -243,8 +243,8 @@ void handle_port_absent_message(federate_info_t* sending_federate, unsigned char void handle_timed_message(federate_info_t* sending_federate, unsigned char* buffer) { size_t header_size = 1 + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint32_t) + sizeof(int64_t) + sizeof(uint32_t); // Read the header, minus the first byte which has already been read. - read_from_socket_fail_on_error(&sending_federate->socket, header_size - 1, &(buffer[1]), - "RTI failed to read the timed message header from remote federate."); + read_from_net_fail_on_error(sending_federate->net, header_size - 1, &(buffer[1]), + "RTI failed to read the timed message header from remote federate."); // Extract the header information. of the sender uint16_t reactor_port_id; uint16_t federate_id; @@ -272,8 +272,8 @@ void handle_timed_message(federate_info_t* sending_federate, unsigned char* buff sending_federate->enclave.id, federate_id, reactor_port_id, intended_tag.time - lf_time_start(), intended_tag.microstep); - read_from_socket_fail_on_error(&sending_federate->socket, bytes_to_read, &(buffer[header_size]), - "RTI failed to read timed message from federate %d.", federate_id); + read_from_net_fail_on_error(sending_federate->net, bytes_to_read, &(buffer[header_size]), + "RTI failed to read timed message from federate %d.", federate_id); size_t bytes_read = bytes_to_read + header_size; // Following only works for string messages. // LF_PRINT_DEBUG("Message received by RTI: %s.", buffer + header_size); @@ -283,12 +283,12 @@ void handle_timed_message(federate_info_t* sending_federate, unsigned char* buff } // Need to acquire the mutex lock to ensure that the thread handling - // messages coming from the socket connected to the destination does not + // messages coming from the network abstraction connected to the destination does not // issue a TAG before this message has been forwarded. LF_MUTEX_LOCK(&rti_mutex); // If the destination federate is no longer connected, issue a warning, - // remove the message from the socket and return. + // remove the message from the network abstraction and return. federate_info_t* fed = GET_FED_INFO(federate_id); if (fed->enclave.state == NOT_CONNECTED) { lf_print_warning("RTI: Destination federate %d is no longer connected. Dropping message.", federate_id); @@ -308,8 +308,7 @@ void handle_timed_message(federate_info_t* sending_federate, unsigned char* buff if (bytes_to_read > FED_COM_BUFFER_SIZE) { bytes_to_read = FED_COM_BUFFER_SIZE; } - read_from_socket_fail_on_error(&sending_federate->socket, bytes_to_read, buffer, - "RTI failed to clear message chunks."); + read_from_net_fail_on_error(sending_federate->net, bytes_to_read, buffer, "RTI failed to clear message chunks."); total_bytes_read += bytes_to_read; } LF_MUTEX_UNLOCK(&rti_mutex); @@ -330,8 +329,8 @@ void handle_timed_message(federate_info_t* sending_federate, unsigned char* buff tracepoint_rti_to_federate(send_TAGGED_MSG, federate_id, &intended_tag); } - write_to_socket_fail_on_error(&fed->socket, bytes_read, buffer, &rti_mutex, - "RTI failed to forward message to federate %d.", federate_id); + write_to_net_fail_on_error(fed->net, bytes_read, buffer, &rti_mutex, "RTI failed to forward message to federate %d.", + federate_id); // The message length may be longer than the buffer, // in which case we have to handle it in chunks. @@ -342,16 +341,14 @@ void handle_timed_message(federate_info_t* sending_federate, unsigned char* buff if (bytes_to_read > FED_COM_BUFFER_SIZE) { bytes_to_read = FED_COM_BUFFER_SIZE; } - read_from_socket_fail_on_error(&sending_federate->socket, bytes_to_read, buffer, - "RTI failed to read message chunks."); + read_from_net_fail_on_error(sending_federate->net, bytes_to_read, buffer, "RTI failed to read message chunks."); total_bytes_read += bytes_to_read; // FIXME: a mutex needs to be held for this so that other threads // do not write to destination_socket and cause interleaving. However, // holding the rti_mutex might be very expensive. Instead, each outgoing - // socket should probably have its own mutex. - write_to_socket_fail_on_error(&fed->socket, bytes_to_read, buffer, &rti_mutex, - "RTI failed to send message chunks."); + // network abstraction should probably have its own mutex. + write_to_net_fail_on_error(fed->net, bytes_to_read, buffer, &rti_mutex, "RTI failed to send message chunks."); } // Record this in-transit message in federate's in-transit message queue. @@ -380,9 +377,9 @@ void handle_timed_message(federate_info_t* sending_federate, unsigned char* buff void handle_latest_tag_confirmed(federate_info_t* fed) { unsigned char buffer[sizeof(int64_t) + sizeof(uint32_t)]; - read_from_socket_fail_on_error(&fed->socket, sizeof(int64_t) + sizeof(uint32_t), buffer, - "RTI failed to read the content of the logical tag complete from federate %d.", - fed->enclave.id); + read_from_net_fail_on_error(fed->net, sizeof(int64_t) + sizeof(uint32_t), buffer, + "RTI failed to read the content of the logical tag complete from federate %d.", + fed->enclave.id); tag_t completed = extract_tag(buffer); if (rti_remote->base.tracing_enabled) { tracepoint_rti_from_federate(receive_LTC, fed->enclave.id, &completed); @@ -398,9 +395,9 @@ void handle_latest_tag_confirmed(federate_info_t* fed) { void handle_next_event_tag(federate_info_t* fed) { unsigned char buffer[sizeof(int64_t) + sizeof(uint32_t)]; - read_from_socket_fail_on_error(&fed->socket, sizeof(int64_t) + sizeof(uint32_t), buffer, - "RTI failed to read the content of the next event tag from federate %d.", - fed->enclave.id); + read_from_net_fail_on_error(fed->net, sizeof(int64_t) + sizeof(uint32_t), buffer, + "RTI failed to read the content of the next event tag from federate %d.", + fed->enclave.id); // Acquire a mutex lock to ensure that this state does not change while a // message is in transport or being used to determine a TAG. @@ -456,8 +453,8 @@ static void broadcast_stop_time_to_federates_locked() { if (rti_remote->base.tracing_enabled) { tracepoint_rti_to_federate(send_STOP_GRN, fed->enclave.id, &rti_remote->base.max_stop_tag); } - write_to_socket_fail_on_error(&fed->socket, MSG_TYPE_STOP_GRANTED_LENGTH, outgoing_buffer, &rti_mutex, - "RTI failed to send MSG_TYPE_STOP_GRANTED message to federate %d.", fed->enclave.id); + write_to_net_fail_on_error(fed->net, MSG_TYPE_STOP_GRANTED_LENGTH, outgoing_buffer, &rti_mutex, + "RTI failed to send MSG_TYPE_STOP_GRANTED message to federate %d.", fed->enclave.id); } LF_PRINT_LOG("RTI sent to federates MSG_TYPE_STOP_GRANTED with tag " PRINTF_TAG, @@ -509,9 +506,9 @@ void handle_stop_request_message(federate_info_t* fed) { size_t bytes_to_read = MSG_TYPE_STOP_REQUEST_LENGTH - 1; unsigned char buffer[bytes_to_read]; - read_from_socket_fail_on_error(&fed->socket, bytes_to_read, buffer, - "RTI failed to read the MSG_TYPE_STOP_REQUEST payload from federate %d.", - fed->enclave.id); + read_from_net_fail_on_error(fed->net, bytes_to_read, buffer, + "RTI failed to read the MSG_TYPE_STOP_REQUEST payload from federate %d.", + fed->enclave.id); // Extract the proposed stop tag for the federate tag_t proposed_stop_tag = extract_tag(buffer); @@ -576,9 +573,8 @@ void handle_stop_request_message(federate_info_t* fed) { if (rti_remote->base.tracing_enabled) { tracepoint_rti_to_federate(send_STOP_REQ, f->enclave.id, &rti_remote->base.max_stop_tag); } - write_to_socket_fail_on_error(&f->socket, MSG_TYPE_STOP_REQUEST_LENGTH, stop_request_buffer, &rti_mutex, - "RTI failed to forward MSG_TYPE_STOP_REQUEST message to federate %d.", - f->enclave.id); + write_to_net_fail_on_error(f->net, MSG_TYPE_STOP_REQUEST_LENGTH, stop_request_buffer, &rti_mutex, + "RTI failed to forward MSG_TYPE_STOP_REQUEST message to federate %d.", f->enclave.id); } } LF_PRINT_LOG("RTI forwarded to federates MSG_TYPE_STOP_REQUEST with tag (" PRINTF_TIME ", %u).", @@ -589,9 +585,9 @@ void handle_stop_request_message(federate_info_t* fed) { void handle_stop_request_reply(federate_info_t* fed) { size_t bytes_to_read = MSG_TYPE_STOP_REQUEST_REPLY_LENGTH - 1; unsigned char buffer_stop_time[bytes_to_read]; - read_from_socket_fail_on_error(&fed->socket, bytes_to_read, buffer_stop_time, - "RTI failed to read the reply to MSG_TYPE_STOP_REQUEST message from federate %d.", - fed->enclave.id); + read_from_net_fail_on_error(fed->net, bytes_to_read, buffer_stop_time, + "RTI failed to read the reply to MSG_TYPE_STOP_REQUEST message from federate %d.", + fed->enclave.id); tag_t federate_stop_tag = extract_tag(buffer_stop_time); @@ -619,8 +615,7 @@ void handle_address_query(uint16_t fed_id) { // Use buffer both for reading and constructing the reply. // The length is what is needed for the reply. unsigned char buffer[1 + sizeof(int32_t)]; - read_from_socket_fail_on_error(&fed->socket, sizeof(uint16_t), (unsigned char*)buffer, - "Failed to read address query."); + read_from_net_fail_on_error(fed->net, sizeof(uint16_t), (unsigned char*)buffer, "Failed to read address query."); uint16_t remote_fed_id = extract_uint16(buffer); if (rti_remote->base.tracing_enabled) { @@ -639,20 +634,39 @@ void handle_address_query(uint16_t fed_id) { // Encode the port number. federate_info_t* remote_fed = GET_FED_INFO(remote_fed_id); - // Send the port number (which could be -1). + int32_t server_port; + uint32_t* ip_address; + char* server_host_name; + LF_MUTEX_LOCK(&rti_mutex); - encode_int32(remote_fed->server_port, (unsigned char*)&buffer[1]); - write_to_socket_fail_on_error(&fed->socket, sizeof(int32_t) + 1, (unsigned char*)buffer, &rti_mutex, - "Failed to write port number to socket of federate %d.", fed_id); - - // Send the server IP address to federate. - write_to_socket_fail_on_error(&fed->socket, sizeof(remote_fed->server_ip_addr), - (unsigned char*)&remote_fed->server_ip_addr, &rti_mutex, - "Failed to write ip address to socket of federate %d.", fed_id); + // Check if the RTI has initialized the remote federate's network abstraction. + if (remote_fed->net == NULL) { + // RTI has not set up the remote federate. Respond with -1 to indicate an unknown port number. + server_port = -1; + uint32_t temp = 0; + ip_address = &temp; + server_host_name = "localhost"; + } else { + // The network abstraction is initialized, but the RTI might still not know the port number. This can happen if the + // RTI has not yet received a MSG_TYPE_ADDRESS_ADVERTISEMENT message from the remote federate. In such cases, the + // returned port number might still be -1. + server_port = get_server_port(remote_fed->net); + ip_address = (uint32_t*)get_ip_addr(remote_fed->net); + server_host_name = get_server_hostname(remote_fed->net); + } + + encode_int32(server_port, (unsigned char*)&buffer[1]); + + // Send the port number (which could be -1) and the server IP address to federate. + write_to_net_fail_on_error(fed->net, 1 + sizeof(int32_t), (unsigned char*)buffer, &rti_mutex, + "Failed to write port number to network abstraction of federate %d.", fed_id); + + write_to_net_fail_on_error(fed->net, sizeof(uint32_t), (unsigned char*)ip_address, &rti_mutex, + "Failed to write ip address to network abstraction of federate %d.", fed_id); LF_MUTEX_UNLOCK(&rti_mutex); - LF_PRINT_DEBUG("Replied to address query from federate %d with address %s:%d.", fed_id, remote_fed->server_hostname, - remote_fed->server_port); + LF_PRINT_DEBUG("Replied to address query from federate %d with address %s:%d.", fed_id, server_host_name, + server_port); } void handle_address_ad(uint16_t federate_id) { @@ -661,15 +675,15 @@ void handle_address_ad(uint16_t federate_id) { // connections to other federates int32_t server_port = -1; unsigned char buffer[sizeof(int32_t)]; - read_from_socket_fail_on_error(&fed->socket, sizeof(int32_t), (unsigned char*)buffer, - "Error reading port data from federate %d.", federate_id); + read_from_net_fail_on_error(fed->net, sizeof(int32_t), (unsigned char*)buffer, + "Error reading port data from federate %d.", federate_id); server_port = extract_int32(buffer); assert(server_port < 65536); LF_MUTEX_LOCK(&rti_mutex); - fed->server_port = server_port; + set_server_port(fed->net, server_port); LF_MUTEX_UNLOCK(&rti_mutex); LF_PRINT_LOG("Received address advertisement with port %d from federate %d.", server_port, federate_id); @@ -680,9 +694,9 @@ void handle_address_ad(uint16_t federate_id) { void handle_timestamp(federate_info_t* my_fed) { unsigned char buffer[sizeof(int64_t)]; - // Read bytes from the socket. We need 8 bytes. - read_from_socket_fail_on_error(&my_fed->socket, sizeof(int64_t), (unsigned char*)&buffer, - "ERROR reading timestamp from federate %d.\n", my_fed->enclave.id); + // Read bytes from the network abstraction. We need 8 bytes. + read_from_net_fail_on_error(my_fed->net, sizeof(int64_t), (unsigned char*)&buffer, + "ERROR reading timestamp from federate %d.\n", my_fed->enclave.id); int64_t timestamp = swap_bytes_if_big_endian_int64(*((int64_t*)(&buffer))); if (rti_remote->base.tracing_enabled) { @@ -723,7 +737,7 @@ void handle_timestamp(federate_info_t* my_fed) { tag_t tag = {.time = start_time, .microstep = 0}; tracepoint_rti_to_federate(send_TIMESTAMP, my_fed->enclave.id, &tag); } - if (write_to_socket(my_fed->socket, MSG_TYPE_TIMESTAMP_LENGTH, start_time_buffer)) { + if (write_to_net(my_fed->net, MSG_TYPE_TIMESTAMP_LENGTH, start_time_buffer)) { lf_print_error("Failed to send the starting time to federate %d.", my_fed->enclave.id); } @@ -739,8 +753,9 @@ void handle_timestamp(federate_info_t* my_fed) { void send_physical_clock(unsigned char message_type, federate_info_t* fed, socket_type_t socket_type) { if (fed->enclave.state == NOT_CONNECTED) { - lf_print_warning("Clock sync: RTI failed to send physical time to federate %d. Socket not connected.\n", - fed->enclave.id); + lf_print_warning( + "Clock sync: RTI failed to send physical time to federate %d. network abstraction not connected.\n", + fed->enclave.id); return; } unsigned char buffer[sizeof(int64_t) + 1]; @@ -748,9 +763,8 @@ void send_physical_clock(unsigned char message_type, federate_info_t* fed, socke int64_t current_physical_time = lf_time_physical(); encode_int64(current_physical_time, &(buffer[1])); - // Send the message if (socket_type == UDP) { - // FIXME: UDP_addr is never initialized. + // Send using UDP LF_PRINT_DEBUG("Clock sync: RTI sending UDP message type %u.", buffer[0]); ssize_t bytes_written = sendto(rti_remote->socket_descriptor_UDP, buffer, 1 + sizeof(int64_t), 0, (struct sockaddr*)&fed->UDP_addr, sizeof(fed->UDP_addr)); @@ -759,11 +773,12 @@ void send_physical_clock(unsigned char message_type, federate_info_t* fed, socke strerror(errno)); return; } - } else if (socket_type == TCP) { - LF_PRINT_DEBUG("Clock sync: RTI sending TCP message type %u.", buffer[0]); + } else { + // Send using network abstraction. + LF_PRINT_DEBUG("Clock sync: RTI sending message type %u.", buffer[0]); LF_MUTEX_LOCK(&rti_mutex); - write_to_socket_fail_on_error(&fed->socket, 1 + sizeof(int64_t), buffer, &rti_mutex, - "Clock sync: RTI failed to send physical time to federate %d.", fed->enclave.id); + write_to_net_fail_on_error(fed->net, 1 + sizeof(int64_t), buffer, &rti_mutex, + "Clock sync: RTI failed to send physical time to federate %d.", fed->enclave.id); LF_MUTEX_UNLOCK(&rti_mutex); } LF_PRINT_DEBUG("Clock sync: RTI sent PHYSICAL_TIME_SYNC_MESSAGE with timestamp " PRINTF_TIME " to federate %d.", @@ -774,12 +789,14 @@ void handle_physical_clock_sync_message(federate_info_t* my_fed, socket_type_t s // Lock the mutex to prevent interference between sending the two // coded probe messages. LF_MUTEX_LOCK(&rti_mutex); - // Reply with a T4 type message - send_physical_clock(MSG_TYPE_CLOCK_SYNC_T4, my_fed, socket_type); - // Send the corresponding coded probe immediately after, - // but only if this is a UDP channel. - if (socket_type == UDP) { - send_physical_clock(MSG_TYPE_CLOCK_SYNC_CODED_PROBE, my_fed, socket_type); + if (socket_type == TCP) { + // Reply with a T4 type message + send_physical_clock(MSG_TYPE_CLOCK_SYNC_T4, my_fed, TCP); + // Send the corresponding coded probe immediately after, + // but only if this is a UDP channel. + } else { + send_physical_clock(MSG_TYPE_CLOCK_SYNC_T4, my_fed, UDP); + send_physical_clock(MSG_TYPE_CLOCK_SYNC_CODED_PROBE, my_fed, UDP); } LF_MUTEX_UNLOCK(&rti_mutex); } @@ -882,7 +899,7 @@ void* clock_synchronization_thread(void* noargs) { * @param my_fed The federate sending a MSG_TYPE_FAILED message. */ static void handle_federate_failed(federate_info_t* my_fed) { - // Nothing more to do. Close the socket and exit. + // Nothing more to do. Close the network abstraction and exit. LF_MUTEX_LOCK(&rti_mutex); if (rti_remote->base.tracing_enabled) { @@ -898,7 +915,7 @@ static void handle_federate_failed(federate_info_t* my_fed) { // Indicate that there will no further events from this federate. my_fed->enclave.next_event = FOREVER_TAG; - shutdown_socket(&my_fed->socket, false); + shutdown_net(my_fed->net, false); // Check downstream federates to see whether they should now be granted a TAG. // To handle cycles, need to create a boolean array to keep @@ -917,13 +934,13 @@ static void handle_federate_failed(federate_info_t* my_fed) { * This function assumes the caller does not hold the mutex. * * @note At this point, the RTI might have outgoing messages to the federate. This - * function thus first performs a shutdown on the socket, which sends an EOF. It then - * waits for the remote socket to be closed before closing the socket itself. + * function thus first performs a shutdown on the network abstraction, which sends an EOF. It then + * waits for the remote network abstraction to be closed before closing the network abstraction itself. * * @param my_fed The federate sending a MSG_TYPE_RESIGN message. */ static void handle_federate_resign(federate_info_t* my_fed) { - // Nothing more to do. Close the socket and exit. + // Nothing more to do. Close the network abstraction and exit. LF_MUTEX_LOCK(&rti_mutex); if (rti_remote->base.tracing_enabled) { @@ -937,7 +954,7 @@ static void handle_federate_resign(federate_info_t* my_fed) { // Indicate that there will no further events from this federate. my_fed->enclave.next_event = FOREVER_TAG; - shutdown_socket(&my_fed->socket, true); + shutdown_net(my_fed->net, true); // Check downstream federates to see whether they should now be granted a TAG. // To handle cycles, need to create a boolean array to keep @@ -961,14 +978,14 @@ void* federate_info_thread_TCP(void* fed) { // Listen for messages from the federate. while (my_fed->enclave.state != NOT_CONNECTED) { // Read no more than one byte to get the message type. - int read_failed = read_from_socket(my_fed->socket, 1, buffer); + int read_failed = read_from_net(my_fed->net, 1, buffer); if (read_failed) { - // Socket is closed - lf_print_error("RTI: Socket to federate %d is closed. Exiting the thread.", my_fed->enclave.id); + // network abstraction is closed + lf_print_error("RTI: network abstraction to federate %d is closed. Exiting the thread.", my_fed->enclave.id); my_fed->enclave.state = NOT_CONNECTED; - // Nothing more to do. Close the socket and exit. - // Prevent multiple threads from closing the same socket at the same time. - shutdown_socket(&my_fed->socket, false); // from unistd.h + // Nothing more to do. Close the network abstraction and exit. + // Prevent multiple threads from closing the same network abstraction at the same time. + shutdown_net(my_fed->net, false); // FIXME: We need better error handling here, but do not stop execution here. break; } @@ -1021,18 +1038,18 @@ void* federate_info_thread_TCP(void* fed) { return NULL; } -void send_reject(int* socket_id, unsigned char error_code) { +void send_reject(net_abstraction_t net_abs, unsigned char error_code) { LF_PRINT_DEBUG("RTI sending MSG_TYPE_REJECT."); unsigned char response[2]; response[0] = MSG_TYPE_REJECT; response[1] = error_code; LF_MUTEX_LOCK(&rti_mutex); // NOTE: Ignore errors on this response. - if (write_to_socket(*socket_id, 2, response)) { - lf_print_warning("RTI failed to write MSG_TYPE_REJECT message on the socket."); + if (write_to_net(net_abs, 2, response)) { + lf_print_warning("RTI failed to write MSG_TYPE_REJECT message on the network abstraction."); } - // Close the socket without reading until EOF. - shutdown_socket(socket_id, false); + // Close the network abstraction without reading until EOF. + shutdown_net(net_abs, false); LF_MUTEX_UNLOCK(&rti_mutex); } @@ -1041,18 +1058,17 @@ void send_reject(int* socket_id, unsigned char error_code) { * a federate ID and a federation ID. If the federation ID * matches this federation, send an MSG_TYPE_ACK and otherwise send * a MSG_TYPE_REJECT message. - * @param socket_id Pointer to the socket on which to listen. - * @param client_fd The socket address. + * @param fed_net Pointer to the network abstraction on which to listen. * @return The federate ID for success or -1 for failure. */ -static int32_t receive_and_check_fed_id_message(int* socket_id) { +static int32_t receive_and_check_fed_id_message(net_abstraction_t fed_net) { // Buffer for message ID, federate ID, and federation ID length. size_t length = 1 + sizeof(uint16_t) + 1; // Message ID, federate ID, length of fedration ID. unsigned char buffer[length]; - // Read bytes from the socket. We need 4 bytes. - if (read_from_socket_close_on_error(socket_id, length, buffer)) { - lf_print_error("RTI failed to read from accepted socket."); + // Read bytes from the network abstraction. We need 4 bytes. + if (read_from_net_close_on_error(fed_net, length, buffer)) { + lf_print_error("RTI failed to read from accepted network abstraction."); return -1; } @@ -1071,12 +1087,12 @@ static int32_t receive_and_check_fed_id_message(int* socket_id) { // of the peer they want to connect to from the RTI. // If the connection is a peer-to-peer connection between two // federates, reject the connection with the WRONG_SERVER error. - send_reject(socket_id, WRONG_SERVER); + send_reject(fed_net, WRONG_SERVER); } else if (buffer[0] == MSG_TYPE_FED_NONCE) { - send_reject(socket_id, RTI_NOT_EXECUTED_WITH_AUTH); + send_reject(fed_net, RTI_NOT_EXECUTED_WITH_AUTH); lf_print_error("RTI not executed with HMAC authentication option using -a or --auth."); } else { - send_reject(socket_id, UNEXPECTED_MESSAGE); + send_reject(fed_net, UNEXPECTED_MESSAGE); } lf_print_error("RTI expected a MSG_TYPE_FED_IDS message. Got %u (see net_common.h).", buffer[0]); return -1; @@ -1089,7 +1105,7 @@ static int32_t receive_and_check_fed_id_message(int* socket_id) { size_t federation_id_length = (size_t)buffer[sizeof(uint16_t) + 1]; char federation_id_received[federation_id_length + 1]; // One extra for null terminator. // Next read the actual federation ID. - if (read_from_socket_close_on_error(socket_id, federation_id_length, (unsigned char*)federation_id_received)) { + if (read_from_net_close_on_error(fed_net, federation_id_length, (unsigned char*)federation_id_received)) { lf_print_error("RTI failed to read federation id from federate %d.", fed_id); return -1; } @@ -1110,7 +1126,7 @@ static int32_t receive_and_check_fed_id_message(int* socket_id) { if (rti_remote->base.tracing_enabled) { tracepoint_rti_to_federate(send_REJECT, fed_id, NULL); } - send_reject(socket_id, FEDERATION_ID_DOES_NOT_MATCH); + send_reject(fed_net, FEDERATION_ID_DOES_NOT_MATCH); return -1; } else { if (fed_id >= rti_remote->base.number_of_scheduling_nodes) { @@ -1119,7 +1135,7 @@ static int32_t receive_and_check_fed_id_message(int* socket_id) { if (rti_remote->base.tracing_enabled) { tracepoint_rti_to_federate(send_REJECT, fed_id, NULL); } - send_reject(socket_id, FEDERATE_ID_OUT_OF_RANGE); + send_reject(fed_net, FEDERATE_ID_OUT_OF_RANGE); return -1; } else { if ((rti_remote->base.scheduling_nodes[fed_id])->state != NOT_CONNECTED) { @@ -1127,7 +1143,7 @@ static int32_t receive_and_check_fed_id_message(int* socket_id) { if (rti_remote->base.tracing_enabled) { tracepoint_rti_to_federate(send_REJECT, fed_id, NULL); } - send_reject(socket_id, FEDERATE_ID_IN_USE); + send_reject(fed_net, FEDERATE_ID_IN_USE); return -1; } } @@ -1136,24 +1152,7 @@ static int32_t receive_and_check_fed_id_message(int* socket_id) { federate_info_t* fed = GET_FED_INFO(fed_id); // The MSG_TYPE_FED_IDS message has the right federation ID. - // Get the peer address from the connected socket_id. Then assign it as the federate's socket server. - struct sockaddr_in peer_addr; - socklen_t addr_len = sizeof(peer_addr); - if (getpeername(*socket_id, (struct sockaddr*)&peer_addr, &addr_len) != 0) { - lf_print_error("RTI failed to get peer address."); - } - fed->server_ip_addr = peer_addr.sin_addr; - -#if LOG_LEVEL >= LOG_LEVEL_DEBUG - // Create the human readable format and copy that into - // the .server_hostname field of the federate. - char str[INET_ADDRSTRLEN + 1]; - inet_ntop(AF_INET, &fed->server_ip_addr, str, INET_ADDRSTRLEN); - strncpy(fed->server_hostname, str, INET_ADDRSTRLEN); - - LF_PRINT_DEBUG("RTI got address %s from federate %d.", fed->server_hostname, fed_id); -#endif - fed->socket = *socket_id; + fed->net = fed_net; // Set the federate's state as pending // because it is waiting for the start time to be @@ -1167,7 +1166,7 @@ static int32_t receive_and_check_fed_id_message(int* socket_id) { tracepoint_rti_to_federate(send_ACK, fed_id, NULL); } LF_MUTEX_LOCK(&rti_mutex); - if (write_to_socket_close_on_error(&fed->socket, 1, &ack_message)) { + if (write_to_net_close_on_error(fed->net, 1, &ack_message)) { LF_MUTEX_UNLOCK(&rti_mutex); lf_print_error("RTI failed to write MSG_TYPE_ACK message to federate %d.", fed_id); return -1; @@ -1184,18 +1183,18 @@ static int32_t receive_and_check_fed_id_message(int* socket_id) { * out the relevant information in the federate's struct. * @return 1 on success and 0 on failure. */ -static int receive_connection_information(int* socket_id, uint16_t fed_id) { +static int receive_connection_information(net_abstraction_t fed_net, uint16_t fed_id) { LF_PRINT_DEBUG("RTI waiting for MSG_TYPE_NEIGHBOR_STRUCTURE from federate %d.", fed_id); unsigned char connection_info_header[MSG_TYPE_NEIGHBOR_STRUCTURE_HEADER_SIZE]; - read_from_socket_fail_on_error(socket_id, MSG_TYPE_NEIGHBOR_STRUCTURE_HEADER_SIZE, connection_info_header, - "RTI failed to read MSG_TYPE_NEIGHBOR_STRUCTURE message header from federate %d.", - fed_id); + read_from_net_fail_on_error(fed_net, MSG_TYPE_NEIGHBOR_STRUCTURE_HEADER_SIZE, connection_info_header, + "RTI failed to read MSG_TYPE_NEIGHBOR_STRUCTURE message header from federate %d.", + fed_id); if (connection_info_header[0] != MSG_TYPE_NEIGHBOR_STRUCTURE) { lf_print_error("RTI was expecting a MSG_TYPE_UDP_PORT message from federate %d. Got %u instead. " "Rejecting federate.", fed_id, connection_info_header[0]); - send_reject(socket_id, UNEXPECTED_MESSAGE); + send_reject(fed_net, UNEXPECTED_MESSAGE); return 0; } else { federate_info_t* fed = GET_FED_INFO(fed_id); @@ -1227,9 +1226,9 @@ static int receive_connection_information(int* socket_id, uint16_t fed_id) { if (connections_info_body_size > 0) { connections_info_body = (unsigned char*)malloc(connections_info_body_size); LF_ASSERT_NON_NULL(connections_info_body); - read_from_socket_fail_on_error(socket_id, connections_info_body_size, connections_info_body, - "RTI failed to read MSG_TYPE_NEIGHBOR_STRUCTURE message body from federate %d.", - fed_id); + read_from_net_fail_on_error(fed_net, connections_info_body_size, connections_info_body, + "RTI failed to read MSG_TYPE_NEIGHBOR_STRUCTURE message body from federate %d.", + fed_id); // Keep track of where we are in the buffer size_t message_head = 0; // First, read the info about upstream federates @@ -1261,23 +1260,23 @@ static int receive_connection_information(int* socket_id, uint16_t fed_id) { * up to perform runtime clock synchronization using the UDP port number * specified in the payload to communicate with the federate's clock * synchronization logic. - * @param socket_id The socket on which to listen. + * @param fed_net The network abstraction on which to listen. * @param fed_id The federate ID. * @return 1 for success, 0 for failure. */ -static int receive_udp_message_and_set_up_clock_sync(int* socket_id, uint16_t fed_id) { +static int receive_udp_message_and_set_up_clock_sync(net_abstraction_t fed_net, uint16_t fed_id) { // Read the MSG_TYPE_UDP_PORT message from the federate regardless of the status of // clock synchronization. This message will tell the RTI whether the federate // is doing clock synchronization, and if it is, what port to use for UDP. LF_PRINT_DEBUG("RTI waiting for MSG_TYPE_UDP_PORT from federate %d.", fed_id); unsigned char response[1 + sizeof(uint16_t)]; - read_from_socket_fail_on_error(socket_id, 1 + sizeof(uint16_t), response, - "RTI failed to read MSG_TYPE_UDP_PORT message from federate %d.", fed_id); + read_from_net_fail_on_error(fed_net, 1 + sizeof(uint16_t), response, + "RTI failed to read MSG_TYPE_UDP_PORT message from federate %d.", fed_id); if (response[0] != MSG_TYPE_UDP_PORT) { lf_print_error("RTI was expecting a MSG_TYPE_UDP_PORT message from federate %d. Got %u instead. " "Rejecting federate.", fed_id, response[0]); - send_reject(socket_id, UNEXPECTED_MESSAGE); + send_reject(fed_net, UNEXPECTED_MESSAGE); return 0; } else { federate_info_t* fed = GET_FED_INFO(fed_id); @@ -1298,15 +1297,15 @@ static int receive_udp_message_and_set_up_clock_sync(int* socket_id, uint16_t fe // Listen for reply message, which should be T3. size_t message_size = 1 + sizeof(uint16_t); unsigned char buffer[message_size]; - read_from_socket_fail_on_error(socket_id, message_size, buffer, "Socket to federate %d unexpectedly closed.", - fed_id); + read_from_net_fail_on_error(fed_net, message_size, buffer, + "network abstraction to federate %d unexpectedly closed.", fed_id); if (buffer[0] == MSG_TYPE_CLOCK_SYNC_T3) { uint16_t fed_id = extract_uint16(&(buffer[1])); LF_PRINT_DEBUG("RTI received T3 clock sync message from federate %d.", fed_id); handle_physical_clock_sync_message(fed, TCP); } else { lf_print_error("Unexpected message %u from federate %d.", buffer[0], fed_id); - send_reject(socket_id, UNEXPECTED_MESSAGE); + send_reject(fed_net, UNEXPECTED_MESSAGE); return 0; } } @@ -1318,7 +1317,7 @@ static int receive_udp_message_and_set_up_clock_sync(int* socket_id, uint16_t fe // Initialize the UDP_addr field of the federate struct fed->UDP_addr.sin_family = AF_INET; fed->UDP_addr.sin_port = htons(federate_UDP_port_number); - fed->UDP_addr.sin_addr = fed->server_ip_addr; + fed->UDP_addr.sin_addr = *get_ip_addr(fed_net); } } else { // Disable clock sync after initial round. @@ -1341,14 +1340,14 @@ static int receive_udp_message_and_set_up_clock_sync(int* socket_id, uint16_t fe /** * Authenticate incoming federate by performing HMAC-based authentication. * - * @param socket Socket for the incoming federate tryting to authenticate. + * @param fed_net network abstraction for the incoming federate tryting to authenticate. * @return True if authentication is successful and false otherwise. */ -static bool authenticate_federate(int* socket) { +static bool authenticate_federate(net_abstraction_t fed_net) { // Wait for MSG_TYPE_FED_NONCE from federate. size_t fed_id_length = sizeof(uint16_t); unsigned char buffer[1 + fed_id_length + NONCE_LENGTH]; - read_from_socket_fail_on_error(socket, 1 + fed_id_length + NONCE_LENGTH, buffer, "Failed to read MSG_TYPE_FED_NONCE"); + read_from_net_fail_on_error(fed_net, 1 + fed_id_length + NONCE_LENGTH, buffer, "Failed to read MSG_TYPE_FED_NONCE"); if (buffer[0] != MSG_TYPE_FED_NONCE) { lf_print_error_and_exit("Received unexpected response %u from the FED (see net_common.h).", buffer[0]); } @@ -1372,13 +1371,13 @@ static bool authenticate_federate(int* socket) { RAND_bytes(rti_nonce, NONCE_LENGTH); memcpy(&sender[1], rti_nonce, NONCE_LENGTH); memcpy(&sender[1 + NONCE_LENGTH], hmac_tag, hmac_length); - if (write_to_socket(*socket, 1 + NONCE_LENGTH + hmac_length, sender)) { + if (write_to_net(fed_net, 1 + NONCE_LENGTH + hmac_length, sender)) { lf_print_error("Failed to send nonce to federate."); } // Wait for MSG_TYPE_FED_RESPONSE unsigned char received[1 + hmac_length]; - read_from_socket_fail_on_error(socket, 1 + hmac_length, received, "Failed to read federate response."); + read_from_net_fail_on_error(fed_net, 1 + hmac_length, received, "Failed to read federate response."); if (received[0] != MSG_TYPE_FED_RESPONSE) { lf_print_error_and_exit("Received unexpected response %u from the federate (see net_common.h).", received[0]); return false; @@ -1397,7 +1396,7 @@ static bool authenticate_federate(int* socket) { if (memcmp(&received[1], rti_tag, hmac_length) != 0) { // Federation IDs do not match. Send back a HMAC_DOES_NOT_MATCH message. lf_print_warning("HMAC authentication failed. Rejecting the federate."); - send_reject(socket, HMAC_DOES_NOT_MATCH); + send_reject(fed_net, HMAC_DOES_NOT_MATCH); return false; } else { LF_PRINT_LOG("Federate's HMAC verified."); @@ -1406,16 +1405,20 @@ static bool authenticate_federate(int* socket) { } #endif -void lf_connect_to_federates(int socket_descriptor) { +void lf_connect_to_federates(net_abstraction_t rti_net) { for (int i = 0; i < rti_remote->base.number_of_scheduling_nodes; i++) { - int socket_id = accept_socket(rti_remote->socket_descriptor_TCP, -1); + net_abstraction_t fed_net = accept_net(rti_net, NULL); + if (fed_net == NULL) { + lf_print_warning("RTI failed to accept the federate."); + return; + } // Wait for the first message from the federate when RTI -a option is on. #ifdef __RTI_AUTH__ if (rti_remote->authentication_enabled) { - if (!authenticate_federate(&socket_id)) { + if (!authenticate_federate(fed_net)) { lf_print_warning("RTI failed to authenticate the incoming federate."); - // Close the socket without reading until EOF. - shutdown_socket(&socket_id, false); + // Close the network abstraction without reading until EOF. + shutdown_net(fed_net, false); // Ignore the federate that failed authentication. i--; continue; @@ -1424,9 +1427,9 @@ void lf_connect_to_federates(int socket_descriptor) { #endif // The first message from the federate should contain its ID and the federation ID. - int32_t fed_id = receive_and_check_fed_id_message(&socket_id); - if (fed_id >= 0 && socket_id >= 0 && receive_connection_information(&socket_id, (uint16_t)fed_id) && - receive_udp_message_and_set_up_clock_sync(&socket_id, (uint16_t)fed_id)) { + int32_t fed_id = receive_and_check_fed_id_message(fed_net); + if (fed_id >= 0 && receive_connection_information(fed_net, (uint16_t)fed_id) && + receive_udp_message_and_set_up_clock_sync(fed_net, (uint16_t)fed_id)) { // Create a thread to communicate with the federate. // This has to be done after clock synchronization is finished @@ -1465,9 +1468,9 @@ void* respond_to_erroneous_connections(void* nothing) { while (true) { // Wait for an incoming connection request. // The following will block until either a federate attempts to connect - // or shutdown_socket(rti->socket_descriptor_TCP) is called. - int socket_id = accept_socket(rti_remote->socket_descriptor_TCP, -1); - if (socket_id < 0) { + // or shutdown_net(rti->rti_net) is called. + net_abstraction_t fed_net = accept_net(rti_remote->rti_net, NULL); + if (fed_net == NULL) { return NULL; } if (rti_remote->all_federates_exited) { @@ -1479,11 +1482,11 @@ void* respond_to_erroneous_connections(void* nothing) { response[0] = MSG_TYPE_REJECT; response[1] = FEDERATION_ID_DOES_NOT_MATCH; // Ignore errors on this response. - if (write_to_socket(socket_id, 2, response)) { + if (write_to_net(fed_net, 2, response)) { lf_print_warning("RTI failed to write FEDERATION_ID_DOES_NOT_MATCH to erroneous incoming connection."); } - // Close the socket without reading until EOF. - shutdown_socket(&socket_id, false); + // Close the network abstraction without reading until EOF. + shutdown_net(fed_net, false); } return NULL; } @@ -1491,40 +1494,41 @@ void* respond_to_erroneous_connections(void* nothing) { void initialize_federate(federate_info_t* fed, uint16_t id) { initialize_scheduling_node(&(fed->enclave), id); fed->requested_stop = false; - fed->socket = -1; // No socket. fed->clock_synchronization_enabled = true; fed->in_transit_message_tags = pqueue_tag_init(10); - strncpy(fed->server_hostname, "localhost", INET_ADDRSTRLEN); - fed->server_ip_addr.s_addr = 0; - fed->server_port = -1; } -int32_t start_rti_server(uint16_t port) { +int start_rti_server() { _lf_initialize_clock(); - // Create the TCP socket server - if (create_server(port, &rti_remote->socket_descriptor_TCP, &rti_remote->final_port_TCP, TCP, true)) { + // Initialize RTI's network abstraction. + rti_remote->rti_net = initialize_net(); + // Set the user specified port to the network abstraction. + set_my_port(rti_remote->rti_net, rti_remote->user_specified_port); + // Create the server + if (create_server(rti_remote->rti_net, true)) { lf_print_error_system_failure("RTI failed to create TCP server: %s.", strerror(errno)); + return -1; }; lf_print("RTI: Listening for federates."); // Create the UDP socket server - // Try to get the rti_remote->final_port_TCP + 1 port if (rti_remote->clock_sync_global_status >= clock_sync_on) { - if (create_server(rti_remote->final_port_TCP + 1, &rti_remote->socket_descriptor_UDP, &rti_remote->final_port_UDP, - UDP, true)) { + if (create_socket_server(DEFAULT_UDP_PORT, &rti_remote->socket_descriptor_UDP, &rti_remote->final_port_UDP, UDP, + false)) { lf_print_error_system_failure("RTI failed to create UDP server: %s.", strerror(errno)); + return -1; } } - return rti_remote->socket_descriptor_TCP; + return 0; } -void wait_for_federates(int socket_descriptor) { +void wait_for_federates() { // Wait for connections from federates and create a thread for each. - lf_connect_to_federates(socket_descriptor); + lf_connect_to_federates(rti_remote->rti_net); // All federates have connected. lf_print("RTI: All expected federates have connected. Starting execution."); - // The socket server will not continue to accept connections after all the federates + // The network abstraction server will not continue to accept connections after all the federates // have joined. // In case some other federation's federates are trying to join the wrong // federation, need to respond. Start a separate thread to do that. @@ -1543,12 +1547,13 @@ void wait_for_federates(int socket_descriptor) { rti_remote->all_federates_exited = true; - // Shutdown and close the socket that is listening for incoming connections + // Shutdown and close the network abstraction that is listening for incoming connections // so that the accept() call in respond_to_erroneous_connections returns. // That thread should then check rti->all_federates_exited and it should exit. - shutdown_socket(&socket_descriptor, false); + shutdown_net(rti_remote->rti_net, false); if (rti_remote->socket_descriptor_UDP > 0) { + // UDP only uses sockets. shutdown_socket(&rti_remote->socket_descriptor_UDP, false); } } @@ -1571,8 +1576,6 @@ void initialize_RTI(rti_remote_t* rti) { rti_remote->all_federates_exited = false; rti_remote->federation_id = "Unidentified Federation"; rti_remote->user_specified_port = 0; - rti_remote->final_port_TCP = 0; - rti_remote->socket_descriptor_TCP = -1; rti_remote->final_port_UDP = UINT16_MAX; rti_remote->socket_descriptor_UDP = -1; rti_remote->clock_sync_global_status = clock_sync_init; diff --git a/core/federated/RTI/rti_remote.h b/core/federated/RTI/rti_remote.h index 52dd0e70e..58ac3b60a 100644 --- a/core/federated/RTI/rti_remote.h +++ b/core/federated/RTI/rti_remote.h @@ -15,13 +15,6 @@ #ifndef RTI_REMOTE_H #define RTI_REMOTE_H -#include -#include // Provides select() function to read from multiple sockets. -#include // Defines struct sockaddr_in -#include // inet_ntop & inet_pton -#include // Defines read(), write(), and close() -#include // Defines bzero(). - #include "rti_common.h" #ifdef __RTI_AUTH__ @@ -31,7 +24,7 @@ #include "lf_types.h" #include "pqueue_tag.h" -#include "socket_common.h" +#include "net_abstraction.h" /** * @brief Time allowed for federates to reply to stop request. @@ -60,8 +53,8 @@ typedef struct federate_info_t { bool requested_stop; /** @brief The ID of the thread handling communication with this federate. */ lf_thread_t thread_id; - /** @brief The TCP socket descriptor for communicating with this federate. */ - int socket; + /** @brief The network abstraction for communicating with this federate. */ + net_abstraction_t net; /** @brief The UDP address for the federate. */ struct sockaddr_in UDP_addr; /** @brief Indicates the status of clock synchronization for this federate. Enabled by default. */ @@ -69,13 +62,6 @@ typedef struct federate_info_t { /** @brief Record of in-transit messages to this federate that are not yet processed. This record is ordered based on * the time value of each message for a more efficient access. */ pqueue_tag_t* in_transit_message_tags; - /** @brief Human-readable IP address of the federate's socket server. */ - char server_hostname[INET_ADDRSTRLEN]; - /** @brief Port number of the socket server of the federate. The port number will be -1 if there is no server or if - * the RTI has not been informed of the port number. */ - int32_t server_port; - /** @brief Information about the IP address of the socket server of the federate. */ - struct in_addr server_ip_addr; } federate_info_t; /** @@ -126,21 +112,22 @@ typedef struct rti_remote_t { */ const char* federation_id; - /** @brief The desired port specified by the user on the command line. */ + /** @brief The desired port specified by the user on the command line. + * This should be not moved to the net_abstraction_t, because the user can configure this as -p or --port. + */ uint16_t user_specified_port; - /** @brief The final port number that the TCP socket server ends up using. */ - uint16_t final_port_TCP; - - /** @brief The TCP socket descriptor for the socket server. */ - int socket_descriptor_TCP; - /** @brief The final port number that the UDP socket server ends up using. */ uint16_t final_port_UDP; /** @brief The UDP socket descriptor for the socket server. */ int socket_descriptor_UDP; + /** + * The rti's network abstraction. + */ + net_abstraction_t rti_net; + /** @brief Thread performing PTP clock sync sessions periodically. */ lf_thread_t clock_thread; @@ -359,13 +346,13 @@ void* clock_synchronization_thread(void* noargs); void* federate_info_thread_TCP(void* fed); /** - * @brief Send a MSG_TYPE_REJECT message to the specified socket and close the socket. + * @brief Send a MSG_TYPE_REJECT message to the specified channel and close the channel. * @ingroup RTI * - * @param socket_id Pointer to the socket ID. + * @param net_abs Pointer to the network abstraction. * @param error_code An error code. */ -void send_reject(int* socket_id, unsigned char error_code); +void send_reject(net_abstraction_t net_abs, unsigned char error_code); /** * @brief Wait for one incoming connection request from each federate, @@ -374,9 +361,9 @@ void send_reject(int* socket_id, unsigned char error_code); * * Return when all federates have connected. * - * @param socket_descriptor The socket on which to accept connections. + * @param rti_net The rti's network abstraction on which to accept connections. */ -void lf_connect_to_federates(int socket_descriptor); +void lf_connect_to_federates(net_abstraction_t rti_net); /** * @brief Thread to respond to new connections, which could be federates of other federations @@ -403,7 +390,7 @@ void initialize_federate(federate_info_t* fed, uint16_t id); * * @param port The port on which to listen for socket connections, or 0 to use the default port range. */ -int32_t start_rti_server(uint16_t port); +int start_rti_server(); /** * @brief Start the runtime infrastructure (RTI) interaction with the federates and wait for the federates to exit. @@ -411,7 +398,7 @@ int32_t start_rti_server(uint16_t port); * * @param socket_descriptor The socket descriptor returned by start_rti_server(). */ -void wait_for_federates(int socket_descriptor); +void wait_for_federates(); /** * @brief Print a usage message. diff --git a/core/federated/clock-sync.c b/core/federated/clock-sync.c index 2f30fad87..b67a9580a 100644 --- a/core/federated/clock-sync.c +++ b/core/federated/clock-sync.c @@ -170,7 +170,7 @@ uint16_t setup_clock_synchronization_with_rti() { return port_to_return; } -void synchronize_initial_physical_clock_with_rti(int* rti_socket_TCP) { +void synchronize_initial_physical_clock_with_rti(net_abstraction_t rti_net) { LF_PRINT_DEBUG("Waiting for initial clock synchronization messages from the RTI."); size_t message_size = 1 + sizeof(instant_t); @@ -178,9 +178,9 @@ void synchronize_initial_physical_clock_with_rti(int* rti_socket_TCP) { for (int i = 0; i < _LF_CLOCK_SYNC_EXCHANGES_PER_INTERVAL; i++) { // The first message expected from the RTI is MSG_TYPE_CLOCK_SYNC_T1 - read_from_socket_fail_on_error(rti_socket_TCP, message_size, buffer, - "Federate %d did not get the initial clock synchronization message T1 from the RTI.", - _lf_my_fed_id); + read_from_net_fail_on_error(rti_net, message_size, buffer, + "Federate %d did not get the initial clock synchronization message T1 from the RTI.", + _lf_my_fed_id); // Get local physical time before doing anything else. instant_t receive_time = lf_time_physical(); @@ -192,14 +192,14 @@ void synchronize_initial_physical_clock_with_rti(int* rti_socket_TCP) { // Handle the message and send a reply T3 message. // NOTE: No need to acquire the mutex lock during initialization because only // one thread is running. - if (handle_T1_clock_sync_message(buffer, *rti_socket_TCP, receive_time) != 0) { + if (handle_T1_clock_sync_message(buffer, (void*)rti_net, receive_time, false) != 0) { lf_print_error_and_exit("Initial clock sync: Failed to send T3 reply to RTI."); } // Next message from the RTI is required to be MSG_TYPE_CLOCK_SYNC_T4 - read_from_socket_fail_on_error(rti_socket_TCP, message_size, buffer, - "Federate %d did not get the clock synchronization message T4 from the RTI.", - _lf_my_fed_id); + read_from_net_fail_on_error(rti_net, message_size, buffer, + "Federate %d did not get the clock synchronization message T4 from the RTI.", + _lf_my_fed_id); // Check that this is the T4 message. if (buffer[0] != MSG_TYPE_CLOCK_SYNC_T4) { @@ -207,13 +207,13 @@ void synchronize_initial_physical_clock_with_rti(int* rti_socket_TCP) { } // Handle the message. - handle_T4_clock_sync_message(buffer, *rti_socket_TCP, receive_time); + handle_T4_clock_sync_message(buffer, (void*)rti_net, receive_time, false); } LF_PRINT_LOG("Finished initial clock synchronization with the RTI."); } -int handle_T1_clock_sync_message(unsigned char* buffer, int socket, instant_t t2) { +int handle_T1_clock_sync_message(unsigned char* buffer, void* socket_or_net, instant_t t2, socket_type_t socket_type) { // Extract the payload instant_t t1 = extract_int64(&(buffer[1])); @@ -233,7 +233,11 @@ int handle_T1_clock_sync_message(unsigned char* buffer, int socket, instant_t t2 // Write the reply to the socket. LF_PRINT_DEBUG("Sending T3 message to RTI."); - if (write_to_socket(socket, 1 + sizeof(uint16_t), reply_buffer)) { + int result = (socket_type == UDP) + ? write_to_socket(*(int*)socket_or_net, 1 + sizeof(uint16_t), reply_buffer) + : write_to_net((net_abstraction_t)socket_or_net, 1 + sizeof(uint16_t), reply_buffer); + + if (result) { lf_print_error("Clock sync: Failed to send T3 message to RTI."); return -1; } @@ -245,7 +249,7 @@ int handle_T1_clock_sync_message(unsigned char* buffer, int socket, instant_t t2 return 0; } -void handle_T4_clock_sync_message(unsigned char* buffer, int socket, instant_t r4) { +void handle_T4_clock_sync_message(unsigned char* buffer, void* socket_or_net, instant_t r4, socket_type_t socket_type) { // Increment the number of received T4 messages _lf_rti_socket_stat.received_T4_messages_in_current_sync_window++; @@ -277,10 +281,10 @@ void handle_T4_clock_sync_message(unsigned char* buffer, int socket, instant_t r // If the socket is _lf_rti_socket_UDP, then // after sending T4, the RTI sends a "coded probe" message, // which can be used to filter out noise. - if (socket == _lf_rti_socket_UDP) { + if (socket_type == UDP) { // Read the coded probe message. // We can reuse the same buffer. - int read_failed = read_from_socket(socket, 1 + sizeof(instant_t), buffer); + int read_failed = read_from_socket(*(int*)socket_or_net, 1 + sizeof(instant_t), buffer); instant_t r5 = lf_time_physical(); @@ -311,17 +315,15 @@ void handle_T4_clock_sync_message(unsigned char* buffer, int socket, instant_t r _lf_rti_socket_stat.received_T4_messages_in_current_sync_window--; return; } - // Apply a jitter attenuator to the estimated clock error to prevent - // large jumps in the underlying clock. - // Note that estimated_clock_error is calculated using lf_time_physical() which includes - // the clock sync adjustment. - adjustment = estimated_clock_error / _LF_CLOCK_SYNC_ATTENUATION; - } else { - // Use of TCP socket means we are in the startup phase, so - // rather than adjust the clock offset, we simply set it to the - // estimated error. - adjustment = estimated_clock_error; } + // If use UDP, apply a jitter attenuator to the estimated clock error to prevent + // large jumps in the underlying clock. + // Note that estimated_clock_error is calculated using lf_time_physical() which includes + // the clock sync adjustment. + // Use of TCP socket means we are in the startup phase, so + // rather than adjust the clock offset, we simply set it to the + // estimated error. + adjustment = (socket_type == UDP) ? estimated_clock_error / _LF_CLOCK_SYNC_ATTENUATION : estimated_clock_error; #ifdef _LF_CLOCK_SYNC_COLLECT_STATS // Enabled by default // Update RTI's socket stats @@ -437,7 +439,7 @@ static void* listen_to_rti_UDP_thread(void* args) { break; } connected = true; - if (handle_T1_clock_sync_message(buffer, _lf_rti_socket_UDP, receive_time) != 0) { + if (handle_T1_clock_sync_message(buffer, (void*)&_lf_rti_socket_UDP, receive_time, true) != 0) { // Failed to send T3 reply. Wait for the next T1. waiting_for_T1 = true; continue; @@ -450,7 +452,7 @@ static void* listen_to_rti_UDP_thread(void* args) { continue; } } else if (buffer[0] == MSG_TYPE_CLOCK_SYNC_T4) { - handle_T4_clock_sync_message(buffer, _lf_rti_socket_UDP, receive_time); + handle_T4_clock_sync_message(buffer, (void*)&_lf_rti_socket_UDP, receive_time, true); waiting_for_T1 = true; } else { lf_print_warning("Clock sync: Received from RTI an unexpected UDP message type: %u. " diff --git a/core/federated/federate.c b/core/federated/federate.c index af8fcd177..6dabbc9b6 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -13,14 +13,9 @@ #error No support for federated execution on this platform. #endif -#include // inet_ntop & inet_pton -#include // Defines getaddrinfo(), freeaddrinfo() and struct addrinfo. -#include // Defines struct sockaddr_in -#include -#include // Defines read(), write(), and close() -#include // Defines memset(), strnlen(), strncmp(), strncpy() -#include // Defines strerror() - +#include // inet_ntop +#include // Defines memset(), strnlen(), strncmp(), strncpy() +#include // Defines strerror() #include #include // Defined perror(), errno #include // Defines bzero(). @@ -29,6 +24,7 @@ #include "federate.h" #include "net_common.h" #include "net_util.h" +#include "net_abstraction.h" #include "reactor.h" #include "reactor_common.h" #include "reactor_threaded.h" @@ -48,7 +44,7 @@ extern instant_t start_time; extern bool _lf_termination_executed; // Global variables references in federate.h -lf_mutex_t lf_outbound_socket_mutex; +lf_mutex_t lf_outbound_net_mutex; lf_cond_t lf_port_status_changed; @@ -75,13 +71,10 @@ int max_level_allowed_to_advance; * The state of this federate instance. Each executable has exactly one federate instance, * and the _fed global variable refers to that instance. */ -federate_instance_t _fed = {.socket_TCP_RTI = -1, - .number_of_inbound_p2p_connections = 0, - .inbound_socket_listeners = NULL, +federate_instance_t _fed = {.number_of_inbound_p2p_connections = 0, + .inbound_net_listeners = NULL, .number_of_outbound_p2p_connections = 0, .inbound_p2p_handling_thread_id = 0, - .server_socket = -1, - .server_port = -1, .last_TAG = {.time = NEVER, .microstep = 0u}, .is_last_TAG_provisional = false, .has_upstream = false, @@ -102,7 +95,7 @@ federation_metadata_t federation_metadata = { // Static functions (used only internally) /** - * Send a time to the RTI. This acquires the lf_outbound_socket_mutex. + * Send a time to the RTI. This acquires the lf_outbound_net_mutex. * @param type The message type (MSG_TYPE_TIMESTAMP). * @param time The time. */ @@ -117,15 +110,15 @@ static void send_time(unsigned char type, instant_t time) { tag_t tag = {.time = time, .microstep = 0}; tracepoint_federate_to_rti(send_TIMESTAMP, _lf_my_fed_id, &tag); - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, buffer, &lf_outbound_socket_mutex, - "Failed to send time " PRINTF_TIME " to the RTI.", time - start_time); - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_LOCK(&lf_outbound_net_mutex); + write_to_net_fail_on_error(_fed.net_to_RTI, bytes_to_write, buffer, &lf_outbound_net_mutex, + "Failed to send time " PRINTF_TIME " to the RTI.", time - start_time); + LF_MUTEX_UNLOCK(&lf_outbound_net_mutex); } /** * Send a tag to the RTI. - * This function acquires the lf_outbound_socket_mutex. + * This function acquires the lf_outbound_net_mutex. * @param type The message type (MSG_TYPE_NEXT_EVENT_TAG or MSG_TYPE_LATEST_TAG_CONFIRMED). * @param tag The tag. */ @@ -139,30 +132,17 @@ static void send_tag(unsigned char type, tag_t tag) { trace_event_t event_type = (type == MSG_TYPE_NEXT_EVENT_TAG) ? send_NET : send_LTC; // Trace the event when tracing is enabled tracepoint_federate_to_rti(event_type, _lf_my_fed_id, &tag); - - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, buffer, &lf_outbound_socket_mutex, - "Failed to send tag " PRINTF_TAG " to the RTI.", tag.time - start_time, tag.microstep); - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_LOCK(&lf_outbound_net_mutex); + write_to_net_fail_on_error(_fed.net_to_RTI, bytes_to_write, buffer, &lf_outbound_net_mutex, + "Failed to send tag " PRINTF_TAG " to the RTI.", tag.time - start_time, tag.microstep); + LF_MUTEX_UNLOCK(&lf_outbound_net_mutex); } /** - * Return true if either the socket to the RTI is broken or the socket is - * alive and the first unread byte on the socket's queue is MSG_TYPE_FAILED. + * Return true if either the network abstraction to the RTI is broken or the network abstraction is + * alive and the first unread byte on the network abstraction's queue is MSG_TYPE_FAILED. */ -static bool rti_failed() { - unsigned char first_byte; - int socket_id = _fed.socket_TCP_RTI; // Assume atomic read so we don't pass -1 to peek_from_socket. - if (socket_id > 0) { - ssize_t bytes = peek_from_socket(socket_id, &first_byte); - if (bytes < 0 || (bytes == 1 && first_byte == MSG_TYPE_FAILED)) - return true; - else - return false; - } - lf_print_warning("Socket to the RTI is no longer connected."); - return true; -} +static bool rti_failed() { return check_net_closed(_fed.net_to_RTI); } //////////////////////////////// Port Status Handling /////////////////////////////////////// @@ -456,17 +436,17 @@ static bool handle_message_now(environment_t* env, trigger_t* trigger, tag_t int * Handle a message being received from a remote federate. * * This function assumes the caller does not hold the mutex lock. - * @param socket Pointer to the socket to read the message from. + * @param net Pointer to the network abstraction to read the message from. * @param fed_id The sending federate ID or -1 if the centralized coordination. * @return 0 for success, -1 for failure. */ -static int handle_message(int* socket, int fed_id) { +static int handle_message(net_abstraction_t net, int fed_id) { (void)fed_id; // Read the header. size_t bytes_to_read = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint32_t); unsigned char buffer[bytes_to_read]; - if (read_from_socket_close_on_error(socket, bytes_to_read, buffer)) { - // Read failed, which means the socket has been closed between reading the + if (read_from_net_close_on_error(net, bytes_to_read, buffer)) { + // Read failed, which means the network abstraction has been closed between reading the // message ID byte and here. return -1; } @@ -486,8 +466,7 @@ static int handle_message(int* socket, int fed_id) { // Read the payload. // Allocate memory for the message contents. unsigned char* message_contents = (unsigned char*)malloc(length); - if (read_from_socket_close_on_error(socket, length, message_contents)) { - free(message_contents); + if (read_from_net_close_on_error(net, length, message_contents)) { return -1; } // Trace the event when tracing is enabled @@ -511,11 +490,11 @@ static int handle_message(int* socket, int fed_id) { * will not advance to the tag of the message if it is in the future, or * the tag will not advance at all if the tag of the message is * now or in the past. - * @param socket Pointer to the socket to read the message from. + * @param net Pointer to the network abstraction to read the message from. * @param fed_id The sending federate ID or -1 if the centralized coordination. - * @return 0 on successfully reading the message, -1 on failure (e.g. due to socket closed). + * @return 0 on successfully reading the message, -1 on failure (e.g. due to network abstraction closed). */ -static int handle_tagged_message(int* socket, int fed_id) { +static int handle_tagged_message(net_abstraction_t net, int fed_id) { // Environment is always the one corresponding to the top-level scheduling enclave. environment_t* env; _lf_get_environments(&env); @@ -524,7 +503,7 @@ static int handle_tagged_message(int* socket, int fed_id) { size_t bytes_to_read = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint32_t) + sizeof(instant_t) + sizeof(microstep_t); unsigned char buffer[bytes_to_read]; - if (read_from_socket_close_on_error(socket, bytes_to_read, buffer)) { + if (read_from_net_close_on_error(net, bytes_to_read, buffer)) { return -1; // Read failed. } @@ -573,7 +552,7 @@ static int handle_tagged_message(int* socket, int fed_id) { // Read the payload. // Allocate memory for the message contents. unsigned char* message_contents = (unsigned char*)malloc(length); - if (read_from_socket_close_on_error(socket, length, message_contents)) { + if (read_from_net_close_on_error(net, length, message_contents)) { #ifdef FEDERATED_DECENTRALIZED _lf_decrement_tag_barrier_locked(env); #endif @@ -643,13 +622,13 @@ static int handle_tagged_message(int* socket, int fed_id) { if (lf_tag_compare(env->current_tag, env->stop_tag) >= 0 && env->execution_started) { lf_print_error("Received message too late. Already at stop tag.\n" " Current tag is " PRINTF_TAG " and intended tag is " PRINTF_TAG ".\n" - " Discarding message and closing the socket.", + " Discarding message and closing the network connection.", env->current_tag.time - start_time, env->current_tag.microstep, intended_tag.time - start_time, intended_tag.microstep); // Free the allocated memory before returning _lf_done_using(message_token); - // Close socket, reading any incoming data and discarding it. - shutdown_socket(&_fed.sockets_for_inbound_p2p_connections[fed_id], false); + // Close network abstraction, reading any incoming data and discarding it. + shutdown_net(_fed.net_for_inbound_p2p_connections[fed_id], false); LF_MUTEX_UNLOCK(&env->mutex); return -1; } else { @@ -680,14 +659,14 @@ static int handle_tagged_message(int* socket, int fed_id) { * This just sets the last known status tag of the port specified * in the message. * - * @param socket Pointer to the socket to read the message from + * @param net Pointer to the network abstraction to read the message from * @param fed_id The sending federate ID or -1 if the centralized coordination. * @return 0 for success, -1 for failure to complete the read. */ -static int handle_port_absent_message(int* socket, int fed_id) { +static int handle_port_absent_message(net_abstraction_t net, int fed_id) { size_t bytes_to_read = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(instant_t) + sizeof(microstep_t); unsigned char buffer[bytes_to_read]; - if (read_from_socket_close_on_error(socket, bytes_to_read, buffer)) { + if (read_from_net_close_on_error(net, bytes_to_read, buffer)) { return -1; } @@ -724,7 +703,7 @@ static int handle_port_absent_message(int* socket, int fed_id) { * peer federate and calls the appropriate handling function for * each message type. If an error occurs or an EOF is received * from the peer, then this procedure sets the corresponding - * socket in _fed.sockets_for_inbound_p2p_connections + * network abstraction in _fed.net_for_inbound_p2p_connections * to -1 and returns, terminating the thread. * @param _args The remote federate ID (cast to void*). * @param fed_id_ptr A pointer to a uint16_t containing federate ID being listened to. @@ -736,7 +715,7 @@ static void* listen_to_federates(void* _args) { LF_PRINT_LOG("Listening to federate %d.", fed_id); - int* socket_id = &_fed.sockets_for_inbound_p2p_connections[fed_id]; + net_abstraction_t net = _fed.net_for_inbound_p2p_connections[fed_id]; // Buffer for incoming messages. // This does not constrain the message size @@ -745,44 +724,44 @@ static void* listen_to_federates(void* _args) { // Listen for messages from the federate. while (!_lf_termination_executed) { - bool socket_closed = false; + bool net_closed = false; // Read one byte to get the message type. - LF_PRINT_DEBUG("Waiting for a P2P message on socket %d.", *socket_id); + LF_PRINT_DEBUG("Waiting for a P2P message."); bool bad_message = false; - if (read_from_socket_close_on_error(socket_id, 1, buffer)) { - // Socket has been closed. - lf_print("Socket from federate %d is closed.", fed_id); + if (read_from_net_close_on_error(net, 1, buffer)) { + // network abstraction has been closed. + lf_print("network abstraction from federate %d is closed.", fed_id); // Stop listening to this federate. - socket_closed = true; + net_closed = true; } else { - LF_PRINT_DEBUG("Received a P2P message on socket %d of type %d.", *socket_id, buffer[0]); + LF_PRINT_DEBUG("Received a P2P message of type %d.", buffer[0]); switch (buffer[0]) { case MSG_TYPE_P2P_MESSAGE: LF_PRINT_LOG("Received untimed message from federate %d.", fed_id); - if (handle_message(socket_id, fed_id)) { + if (handle_message(net, fed_id)) { // Failed to complete the reading of a message on a physical connection. lf_print_warning("Failed to complete reading of message on physical connection."); - socket_closed = true; + net_closed = true; } break; case MSG_TYPE_P2P_TAGGED_MESSAGE: LF_PRINT_LOG("Received tagged message from federate %d.", fed_id); - if (handle_tagged_message(socket_id, fed_id)) { + if (handle_tagged_message(net, fed_id)) { // P2P tagged messages are only used in decentralized coordination, and - // it is not a fatal error if the socket is closed before the whole message is read. + // it is not a fatal error if the network abstraction is closed before the whole message is read. // But this thread should exit. lf_print_warning("Failed to complete reading of tagged message."); - socket_closed = true; + net_closed = true; } break; case MSG_TYPE_PORT_ABSENT: LF_PRINT_LOG("Received port absent message from federate %d.", fed_id); - if (handle_port_absent_message(socket_id, fed_id)) { + if (handle_port_absent_message(net, fed_id)) { // P2P tagged messages are only used in decentralized coordination, and - // it is not a fatal error if the socket is closed before the whole message is read. + // it is not a fatal error if the network abstraction is closed before the whole message is read. // But this thread should exit. lf_print_warning("Failed to complete reading of tagged message."); - socket_closed = true; + net_closed = true; } break; default: @@ -790,13 +769,13 @@ static void* listen_to_federates(void* _args) { } } if (bad_message) { - lf_print_error("Received erroneous message type: %d. Closing the socket.", buffer[0]); + lf_print_error("Received erroneous message type: %d. Closing the network abstraction.", buffer[0]); // Trace the event when tracing is enabled tracepoint_federate_from_federate(receive_UNIDENTIFIED, _lf_my_fed_id, fed_id, NULL); break; // while loop } - if (socket_closed) { - // For decentralized execution, once this socket is closed, we + if (net_closed) { + // For decentralized execution, once this network abstraction is closed, we // update last known tags of all ports connected to the specified federate to FOREVER_TAG, // which would eliminate the need to wait for STAA to assume an input is absent. mark_inputs_known_absent(fed_id); @@ -808,25 +787,27 @@ static void* listen_to_federates(void* _args) { } /** - * Close the socket that sends outgoing messages to the - * specified federate ID. This function acquires the lf_outbound_socket_mutex mutex lock + * Close the network abstraction that sends outgoing messages to the + * specified federate ID. This function acquires the lf_outbound_net_mutex mutex lock * if _lf_normal_termination is true and otherwise proceeds without the lock. * @param fed_id The ID of the peer federate receiving messages from this * federate, or -1 if the RTI (centralized coordination). */ -static void close_outbound_socket(int fed_id) { +static void close_outbound_net(int fed_id) { assert(fed_id >= 0 && fed_id < NUMBER_OF_FEDERATES); // Close outbound connections, in case they have not closed themselves. // This will result in EOF being sent to the remote federate, except for - // abnormal termination, in which case it will just close the socket. + // abnormal termination, in which case it will just close the network abstraction. if (_lf_normal_termination) { - if (_fed.sockets_for_outbound_p2p_connections[fed_id] >= 0) { - // Close the socket by sending a FIN packet indicating that no further writes + if (_fed.net_for_outbound_p2p_connections[fed_id] != NULL) { + // Close the network abstraction by sending a FIN packet indicating that no further writes // are expected. Then read until we get an EOF indication. - shutdown_socket(&_fed.sockets_for_outbound_p2p_connections[fed_id], true); + shutdown_net(_fed.net_for_outbound_p2p_connections[fed_id], true); + _fed.net_for_outbound_p2p_connections[fed_id] = NULL; } } else { - shutdown_socket(&_fed.sockets_for_outbound_p2p_connections[fed_id], false); + shutdown_net(_fed.net_for_outbound_p2p_connections[fed_id], false); + _fed.net_for_outbound_p2p_connections[fed_id] = NULL; } } @@ -849,14 +830,14 @@ static int perform_hmac_authentication() { memcpy(&fed_hello_buf[1 + fed_id_length], fed_nonce, NONCE_LENGTH); // No mutex needed during startup, hence the NULL argument. - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, message_length, fed_hello_buf, NULL, "Failed to write nonce."); + write_to_net_fail_on_error(_fed.net_to_RTI, message_length, fed_hello_buf, NULL, "Failed to write nonce."); // Check HMAC of received FED_RESPONSE message. unsigned int hmac_length = SHA256_HMAC_LENGTH; size_t federation_id_length = strnlen(federation_metadata.federation_id, 255); unsigned char received[1 + NONCE_LENGTH + hmac_length]; - if (read_from_socket_close_on_error(&_fed.socket_TCP_RTI, 1 + NONCE_LENGTH + hmac_length, received)) { + if (read_from_net_close_on_error(_fed.net_to_RTI, 1 + NONCE_LENGTH + hmac_length, received)) { lf_print_warning("Failed to read RTI response."); return -1; } @@ -890,10 +871,7 @@ static int perform_hmac_authentication() { response[1] = HMAC_DOES_NOT_MATCH; // Ignore errors on writing back. - int socket_id = _fed.socket_TCP_RTI; // Assume atomic read so we don't pass -1 to write_to_socket. - if (socket_id > 0) { - write_to_socket(socket_id, 2, response); - } + write_to_net(_fed.net_to_RTI, 2, response); return -1; } else { LF_PRINT_LOG("HMAC verified."); @@ -907,7 +885,7 @@ static int perform_hmac_authentication() { HMAC(EVP_sha256(), federation_metadata.federation_id, federation_id_length, mac_buf, 1 + NONCE_LENGTH, &sender[1], &hmac_length); - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, 1 + hmac_length, sender, NULL, "Failed to write fed response."); + write_to_net_fail_on_error(_fed.net_to_RTI, 1 + hmac_length, sender, NULL, "Failed to write fed response."); } return 0; } @@ -926,13 +904,13 @@ static instant_t get_start_time_from_rti(instant_t my_physical_time) { // Send the timestamp marker first. send_time(MSG_TYPE_TIMESTAMP, my_physical_time); - // Read bytes from the socket. We need 9 bytes. + // Read bytes from the network abstraction. We need 9 bytes. // Buffer for message ID plus timestamp. size_t buffer_length = 1 + sizeof(instant_t); unsigned char buffer[buffer_length]; - read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, buffer_length, buffer, - "Failed to read MSG_TYPE_TIMESTAMP message from RTI."); + read_from_net_fail_on_error(_fed.net_to_RTI, buffer_length, buffer, + "Failed to read MSG_TYPE_TIMESTAMP message from RTI."); LF_PRINT_DEBUG("Read 9 bytes."); // First byte received is the message ID. @@ -975,8 +953,7 @@ static void handle_tag_advance_grant(void) { size_t bytes_to_read = sizeof(instant_t) + sizeof(microstep_t); unsigned char buffer[bytes_to_read]; - read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, - "Failed to read tag advance grant from RTI."); + read_from_net_fail_on_error(_fed.net_to_RTI, bytes_to_read, buffer, "Failed to read tag advance grant from RTI."); tag_t TAG = extract_tag(buffer); // Trace the event when tracing is enabled @@ -1222,8 +1199,8 @@ static void handle_provisional_tag_advance_grant() { size_t bytes_to_read = sizeof(instant_t) + sizeof(microstep_t); unsigned char buffer[bytes_to_read]; - read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, - "Failed to read provisional tag advance grant from RTI."); + read_from_net_fail_on_error(_fed.net_to_RTI, bytes_to_read, buffer, + "Failed to read provisional tag advance grant from RTI."); tag_t PTAG = extract_tag(buffer); // Trace the event when tracing is enabled @@ -1312,7 +1289,7 @@ static void handle_stop_granted_message() { size_t bytes_to_read = MSG_TYPE_STOP_GRANTED_LENGTH - 1; unsigned char buffer[bytes_to_read]; - read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, "Failed to read stop granted from RTI."); + read_from_net_fail_on_error(_fed.net_to_RTI, bytes_to_read, buffer, "Failed to read stop granted from RTI."); tag_t received_stop_tag = extract_tag(buffer); @@ -1355,7 +1332,7 @@ static void handle_stop_granted_message() { static void handle_stop_request_message() { size_t bytes_to_read = MSG_TYPE_STOP_REQUEST_LENGTH - 1; unsigned char buffer[bytes_to_read]; - read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, "Failed to read stop request from RTI."); + read_from_net_fail_on_error(_fed.net_to_RTI, bytes_to_read, buffer, "Failed to read stop request from RTI."); tag_t tag_to_stop = extract_tag(buffer); // Trace the event when tracing is enabled @@ -1380,10 +1357,10 @@ static void handle_stop_request_message() { // or we have previously sent a stop request to the RTI, // then we have already blocked tag advance in enclaves. // Do not do this twice. The record of whether the first has occurred - // is guarded by the outbound socket mutex. + // is guarded by the outbound network abstraction mutex. // The second is guarded by the global mutex. // Note that the RTI should not send stop requests more than once to federates. - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); + LF_MUTEX_LOCK(&lf_outbound_net_mutex); if (_fed.received_stop_request_from_rti) { LF_PRINT_LOG("Redundant MSG_TYPE_STOP_REQUEST from RTI. Ignoring it."); already_blocked = true; @@ -1392,7 +1369,7 @@ static void handle_stop_request_message() { // prevent lf_request_stop from sending. _fed.received_stop_request_from_rti = true; } - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_UNLOCK(&lf_outbound_net_mutex); if (already_blocked) { // Either we have sent a stop request to the RTI ourselves, @@ -1426,11 +1403,10 @@ static void handle_stop_request_message() { tracepoint_federate_to_rti(send_STOP_REQ_REP, _lf_my_fed_id, &tag_to_stop); // Send the current logical time to the RTI. - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, MSG_TYPE_STOP_REQUEST_REPLY_LENGTH, outgoing_buffer, - &lf_outbound_socket_mutex, - "Failed to send the answer to MSG_TYPE_STOP_REQUEST to RTI."); - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_LOCK(&lf_outbound_net_mutex); + write_to_net_fail_on_error(_fed.net_to_RTI, MSG_TYPE_STOP_REQUEST_REPLY_LENGTH, outgoing_buffer, + &lf_outbound_net_mutex, "Failed to send the answer to MSG_TYPE_STOP_REQUEST to RTI."); + LF_MUTEX_UNLOCK(&lf_outbound_net_mutex); LF_PRINT_DEBUG("Sent MSG_TYPE_STOP_REQUEST_REPLY to RTI with tag " PRINTF_TAG, tag_to_stop.time, tag_to_stop.microstep); @@ -1442,8 +1418,8 @@ static void handle_stop_request_message() { static void handle_downstream_next_event_tag() { size_t bytes_to_read = sizeof(instant_t) + sizeof(microstep_t); unsigned char buffer[bytes_to_read]; - read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, - "Failed to read downstream next event tag from RTI."); + read_from_net_fail_on_error(_fed.net_to_RTI, bytes_to_read, buffer, + "Failed to read downstream next event tag from RTI."); tag_t DNET = extract_tag(buffer); // Trace the event when tracing is enabled @@ -1473,10 +1449,10 @@ static void send_resign_signal() { size_t bytes_to_write = 1; unsigned char buffer[bytes_to_write]; buffer[0] = MSG_TYPE_RESIGN; - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, &(buffer[0]), &lf_outbound_socket_mutex, - "Failed to send MSG_TYPE_RESIGN."); - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_LOCK(&lf_outbound_net_mutex); + write_to_net_fail_on_error(_fed.net_to_RTI, bytes_to_write, &(buffer[0]), &lf_outbound_net_mutex, + "Failed to send MSG_TYPE_RESIGN."); + LF_MUTEX_UNLOCK(&lf_outbound_net_mutex); LF_PRINT_LOG("Resigned."); } @@ -1487,8 +1463,7 @@ static void send_failed_signal() { size_t bytes_to_write = 1; unsigned char buffer[bytes_to_write]; buffer[0] = MSG_TYPE_FAILED; - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, &(buffer[0]), NULL, - "Failed to send MSG_TYPE_FAILED."); + write_to_net_fail_on_error(_fed.net_to_RTI, bytes_to_write, &(buffer[0]), NULL, "Failed to send MSG_TYPE_FAILED."); LF_PRINT_LOG("Failed."); } @@ -1501,11 +1476,11 @@ static void send_failed_signal() { static void handle_rti_failed_message(void) { exit(1); } /** - * Thread that listens for TCP inputs from the RTI. + * Thread that listens for network abstraction inputs from the RTI. * When messages arrive, this calls the appropriate handler. * @param args Ignored */ -static void* listen_to_rti_TCP(void* args) { +static void* listen_to_rti_net(void* args) { (void)args; initialize_lf_thread_id(); // Buffer for incoming messages. @@ -1515,40 +1490,27 @@ static void* listen_to_rti_TCP(void* args) { // Listen for messages from the federate. while (!_lf_termination_executed) { - int socket_id = _fed.socket_TCP_RTI; // Assume atomic read so we don't pass -1 to read_from_socket. - // Check whether the RTI socket is still valid - if (socket_id < 0) { - lf_print_warning("Socket to the RTI unexpectedly closed."); + // Check whether the RTI network abstraction is still valid. + if (_fed.net_to_RTI == NULL) { + lf_print_warning("network abstraction to the RTI unexpectedly closed."); return NULL; } // Read one byte to get the message type. // This will exit if the read fails. - int read_failed = read_from_socket(socket_id, 1, buffer); + int read_failed = read_from_net(_fed.net_to_RTI, 1, buffer); if (read_failed < 0) { - if (errno == ECONNRESET) { - lf_print_error("Socket connection to the RTI was closed by the RTI without" - " properly sending an EOF first. Considering this a soft error."); - // NOTE: If this happens, possibly a new RTI must be elected. - shutdown_socket(&_fed.socket_TCP_RTI, false); - return NULL; - } else { - lf_print_error("Socket connection to the RTI has been broken with error %d: %s." - " The RTI should close connections with an EOF first." - " Considering this a soft error.", - errno, strerror(errno)); - // NOTE: If this happens, possibly a new RTI must be elected. - shutdown_socket(&_fed.socket_TCP_RTI, false); - return NULL; - } + lf_print_error("Connection to the RTI was closed by the RTI with an error. Considering this a soft error."); + shutdown_net(_fed.net_to_RTI, false); + return NULL; } else if (read_failed > 0) { // EOF received. lf_print("Connection to the RTI closed with an EOF."); - shutdown_socket(&_fed.socket_TCP_RTI, false); + shutdown_net(_fed.net_to_RTI, false); return NULL; } switch (buffer[0]) { case MSG_TYPE_TAGGED_MESSAGE: - if (handle_tagged_message(&_fed.socket_TCP_RTI, -1)) { + if (handle_tagged_message(_fed.net_to_RTI, -1)) { // Failures to complete the read of messages from the RTI are fatal. lf_print_error_and_exit("Failed to complete the reading of a message from the RTI."); } @@ -1566,7 +1528,7 @@ static void* listen_to_rti_TCP(void* args) { handle_stop_granted_message(); break; case MSG_TYPE_PORT_ABSENT: - if (handle_port_absent_message(&_fed.socket_TCP_RTI, -1)) { + if (handle_port_absent_message(_fed.net_to_RTI, -1)) { // Failures to complete the read of absent messages from the RTI are fatal. lf_print_error_and_exit("Failed to complete the reading of an absent message from the RTI."); } @@ -1579,10 +1541,10 @@ static void* listen_to_rti_TCP(void* args) { break; case MSG_TYPE_CLOCK_SYNC_T1: case MSG_TYPE_CLOCK_SYNC_T4: - lf_print_error("Federate %d received unexpected clock sync message from RTI on TCP socket.", _lf_my_fed_id); + lf_print_error("Federate %d received unexpected clock sync message from RTI.", _lf_my_fed_id); break; default: - lf_print_error_and_exit("Received from RTI an unrecognized TCP message type: %hhx.", buffer[0]); + lf_print_error_and_exit("Received from RTI an unrecognized message type: %hhx.", buffer[0]); // Trace the event when tracing is enabled tracepoint_federate_from_rti(receive_UNIDENTIFIED, _lf_my_fed_id, NULL); } @@ -1644,7 +1606,7 @@ static bool bounded_NET(tag_t* tag) { // An empty version of this function is code generated for unfederated execution. /** - * Close sockets used to communicate with other federates, if they are open, + * Close network abstractions used to communicate with other federates, if they are open, * and send a MSG_TYPE_RESIGN message to the RTI. This implements the function * defined in reactor.h. For unfederated execution, the code generator * generates an empty implementation. @@ -1655,7 +1617,7 @@ void lf_terminate_execution(environment_t* env) { // For an abnormal termination (e.g. a SIGINT), we need to send a // MSG_TYPE_FAILED message to the RTI, but we should not acquire a mutex. - if (_fed.socket_TCP_RTI >= 0) { + if (_fed.net_to_RTI != NULL) { if (_lf_normal_termination) { tracepoint_federate_to_rti(send_RESIGN, _lf_my_fed_id, &env->current_tag); send_resign_signal(); @@ -1665,45 +1627,45 @@ void lf_terminate_execution(environment_t* env) { } } - LF_PRINT_DEBUG("Closing incoming P2P sockets."); - // Close any incoming P2P sockets that are still open. + LF_PRINT_DEBUG("Closing incoming P2P network abstractions."); + // Close any incoming P2P network abstractions that are still open. for (int i = 0; i < NUMBER_OF_FEDERATES; i++) { - shutdown_socket(&_fed.sockets_for_inbound_p2p_connections[i], false); - // Ignore errors. Mark the socket closed. - _fed.sockets_for_inbound_p2p_connections[i] = -1; + shutdown_net(_fed.net_for_inbound_p2p_connections[i], false); + // Ignore errors. Mark the network abstraction closed. + _fed.net_for_inbound_p2p_connections[i] = NULL; } // Check for all outgoing physical connections in - // _fed.sockets_for_outbound_p2p_connections and - // if the socket ID is not -1, the connection is still open. - // Send an EOF by closing the socket here. + // _fed.net_for_outbound_p2p_connections and + // if the network abstraction ID is not NULL, the connection is still open. + // Send an EOF by closing the network abstraction here. for (int i = 0; i < NUMBER_OF_FEDERATES; i++) { // Close outbound connections, in case they have not closed themselves. // This will result in EOF being sent to the remote federate, except for - // abnormal termination, in which case it will just close the socket. - close_outbound_socket(i); + // abnormal termination, in which case it will just close the network abstraction. + close_outbound_net(i); } - LF_PRINT_DEBUG("Waiting for inbound p2p socket listener threads."); - // Wait for each inbound socket listener thread to close. - if (_fed.number_of_inbound_p2p_connections > 0 && _fed.inbound_socket_listeners != NULL) { + LF_PRINT_DEBUG("Waiting for inbound p2p network abstraction listener threads."); + // Wait for each inbound network abstraction listener thread to close. + if (_fed.number_of_inbound_p2p_connections > 0 && _fed.inbound_net_listeners != NULL) { LF_PRINT_LOG("Waiting for %zu threads listening for incoming messages to exit.", _fed.number_of_inbound_p2p_connections); for (size_t i = 0; i < _fed.number_of_inbound_p2p_connections; i++) { // Ignoring errors here. - lf_thread_join(_fed.inbound_socket_listeners[i], NULL); + lf_thread_join(_fed.inbound_net_listeners[i], NULL); } } - LF_PRINT_DEBUG("Waiting for RTI's socket listener threads."); + LF_PRINT_DEBUG("Waiting for RTI's network abstraction listener threads."); // Wait for the thread listening for messages from the RTI to close. - lf_thread_join(_fed.RTI_socket_listener, NULL); + lf_thread_join(_fed.RTI_net_listener, NULL); // For abnormal termination, there is no need to free memory. if (_lf_normal_termination) { LF_PRINT_DEBUG("Freeing memory occupied by the federate."); - free(_fed.inbound_socket_listeners); + free(_fed.inbound_net_listeners); free(federation_metadata.rti_host); free(federation_metadata.rti_user); } @@ -1732,15 +1694,15 @@ void lf_connect_to_federate(uint16_t remote_federate_id) { // Trace the event when tracing is enabled tracepoint_federate_to_rti(send_ADR_QR, _lf_my_fed_id, NULL); - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, sizeof(uint16_t) + 1, buffer, &lf_outbound_socket_mutex, - "Failed to send address query for federate %d to RTI.", remote_federate_id); - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_LOCK(&lf_outbound_net_mutex); + write_to_net_fail_on_error(_fed.net_to_RTI, sizeof(uint16_t) + 1, buffer, &lf_outbound_net_mutex, + "Failed to send address query for federate %d to RTI.", remote_federate_id); + LF_MUTEX_UNLOCK(&lf_outbound_net_mutex); // Read RTI's response. - read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, sizeof(int32_t) + 1, buffer, - "Failed to read the requested port number for federate %d from RTI.", - remote_federate_id); + read_from_net_fail_on_error(_fed.net_to_RTI, sizeof(int32_t) + 1, buffer, + "Failed to read the requested port number for federate %d from RTI.", + remote_federate_id); if (buffer[0] != MSG_TYPE_ADDRESS_QUERY_REPLY) { // Unexpected reply. Could be that RTI has failed and sent a resignation. @@ -1752,8 +1714,8 @@ void lf_connect_to_federate(uint16_t remote_federate_id) { } port = extract_int32(&buffer[1]); - read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, sizeof(host_ip_addr), (unsigned char*)&host_ip_addr, - "Failed to read the IP address for federate %d from RTI.", remote_federate_id); + read_from_net_fail_on_error(_fed.net_to_RTI, sizeof(host_ip_addr), (unsigned char*)&host_ip_addr, + "Failed to read the IP address for federate %d from RTI.", remote_federate_id); // A reply of -1 for the port means that the RTI does not know // the port number of the remote federate, presumably because the @@ -1773,17 +1735,25 @@ void lf_connect_to_federate(uint16_t remote_federate_id) { char hostname[INET_ADDRSTRLEN]; inet_ntop(AF_INET, &host_ip_addr, hostname, INET_ADDRSTRLEN); - int socket_id = create_real_time_tcp_socket_errexit(); - if (connect_to_socket(socket_id, (const char*)hostname, uport) < 0) { - lf_print_error_and_exit("Failed to connect() to RTI."); + + // Create a network abstraction. + net_abstraction_t net = initialize_net(); + // Set the received host name and port to the network abstraction. + set_server_port(net, uport); + set_server_hostname(net, hostname); + // Create the client network abstraction. + create_client(net); + if (connect_to_net(net) < 0) { + lf_print_error_and_exit("Failed to connect to federate."); } + // Iterate until we either successfully connect or we exceed the CONNECT_TIMEOUT start_connect = lf_time_physical(); while (result < 0 && !_lf_termination_executed) { // Try again after some time if the connection failed. // Note that this should not really happen since the remote federate should be - // accepting socket connections. But possibly it will be busy (in process of accepting - // another socket connection?). Hence, we retry. + // accepting connections. But possibly it will be busy (in process of accepting + // another connection?). Hence, we retry. if (CHECK_TIMEOUT(start_connect, CONNECT_TIMEOUT)) { // If the remote federate is not accepting the connection after CONNECT_TIMEOUT // treat it as a soft error condition and return. @@ -1809,25 +1779,25 @@ void lf_connect_to_federate(uint16_t remote_federate_id) { // Trace the event when tracing is enabled tracepoint_federate_to_federate(send_FED_ID, _lf_my_fed_id, remote_federate_id, NULL); - // No need for a mutex because we have the only handle on the socket. - write_to_socket_fail_on_error(&socket_id, buffer_length, buffer, NULL, "Failed to send fed_id to federate %d.", - remote_federate_id); - write_to_socket_fail_on_error(&socket_id, federation_id_length, (unsigned char*)federation_metadata.federation_id, - NULL, "Failed to send federation id to federate %d.", remote_federate_id); + // No need for a mutex because we have the only handle on the network abstraction. + write_to_net_fail_on_error(net, buffer_length, buffer, NULL, "Failed to send fed_id to federate %d.", + remote_federate_id); + write_to_net_fail_on_error(net, federation_id_length, (unsigned char*)federation_metadata.federation_id, NULL, + "Failed to send federation id to federate %d.", remote_federate_id); - read_from_socket_fail_on_error(&socket_id, 1, (unsigned char*)buffer, - "Failed to read MSG_TYPE_ACK from federate %d in response to sending fed_id.", - remote_federate_id); + read_from_net_fail_on_error(net, 1, (unsigned char*)buffer, + "Failed to read MSG_TYPE_ACK from federate %d in response to sending fed_id.", + remote_federate_id); if (buffer[0] != MSG_TYPE_ACK) { // Get the error code. - read_from_socket_fail_on_error(&socket_id, 1, (unsigned char*)buffer, - "Failed to read error code from federate %d in response to sending fed_id.", - remote_federate_id); + read_from_net_fail_on_error(net, 1, (unsigned char*)buffer, + "Failed to read error code from federate %d in response to sending fed_id.", + remote_federate_id); lf_print_error("Received MSG_TYPE_REJECT message from remote federate (%d).", buffer[0]); result = -1; // Wait ADDRESS_QUERY_RETRY_INTERVAL nanoseconds. lf_sleep(ADDRESS_QUERY_RETRY_INTERVAL); - lf_print_warning("Could not connect to federate %d. Will try again every" PRINTF_TIME "nanoseconds.\n", + lf_print_warning("Could not connect to federate %d. Will try again every " PRINTF_TIME "nanoseconds.\n", remote_federate_id, ADDRESS_QUERY_RETRY_INTERVAL); continue; } else { @@ -1838,8 +1808,8 @@ void lf_connect_to_federate(uint16_t remote_federate_id) { } } // Once we set this variable, then all future calls to close() on this - // socket ID should reset it to -1 within a critical section. - _fed.sockets_for_outbound_p2p_connections[remote_federate_id] = socket_id; + // network abstraction should reset it to NULL within a critical section. + _fed.net_for_outbound_p2p_connections[remote_federate_id] = net; } void lf_connect_to_rti(const char* hostname, int port) { @@ -1849,10 +1819,16 @@ void lf_connect_to_rti(const char* hostname, int port) { hostname = federation_metadata.rti_host ? federation_metadata.rti_host : hostname; port = federation_metadata.rti_port >= 0 ? federation_metadata.rti_port : port; - // Create a socket - _fed.socket_TCP_RTI = create_real_time_tcp_socket_errexit(); - if (connect_to_socket(_fed.socket_TCP_RTI, hostname, port) < 0) { - lf_print_error_and_exit("Failed to connect() to RTI."); + // Create a network abstraction. + _fed.net_to_RTI = initialize_net(); + // Set the user specified host name and port to the network abstraction. + set_server_port(_fed.net_to_RTI, port); + set_server_hostname(_fed.net_to_RTI, hostname); + + // Create the client network abstraction. + create_client(_fed.net_to_RTI); + if (connect_to_net(_fed.net_to_RTI) < 0) { + lf_print_error_and_exit("Failed to connect to RTI."); } instant_t start_connect = lf_time_physical(); @@ -1892,13 +1868,13 @@ void lf_connect_to_rti(const char* hostname, int port) { // Trace the event when tracing is enabled tracepoint_federate_to_rti(send_FED_ID, _lf_my_fed_id, NULL); - // No need for a mutex here because no other threads are writing to this socket. - if (write_to_socket(_fed.socket_TCP_RTI, 2 + sizeof(uint16_t), buffer)) { + // No need for a mutex here because no other threads are writing to this network abstraction. + if (write_to_net(_fed.net_to_RTI, 2 + sizeof(uint16_t), buffer)) { continue; // Try again, possibly on a new port. } // Next send the federation ID itself. - if (write_to_socket(_fed.socket_TCP_RTI, federation_id_length, (unsigned char*)federation_metadata.federation_id)) { + if (write_to_net(_fed.net_to_RTI, federation_id_length, (unsigned char*)federation_metadata.federation_id)) { continue; // Try again. } @@ -1910,7 +1886,7 @@ void lf_connect_to_rti(const char* hostname, int port) { LF_PRINT_DEBUG("Waiting for response to federation ID from the RTI."); - if (read_from_socket(_fed.socket_TCP_RTI, 1, &response)) { + if (read_from_net(_fed.net_to_RTI, 1, &response)) { continue; // Try again. } if (response == MSG_TYPE_REJECT) { @@ -1918,8 +1894,7 @@ void lf_connect_to_rti(const char* hostname, int port) { tracepoint_federate_from_rti(receive_REJECT, _lf_my_fed_id, NULL); // Read one more byte to determine the cause of rejection. unsigned char cause; - read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, 1, &cause, - "Failed to read the cause of rejection by the RTI."); + read_from_net_fail_on_error(_fed.net_to_RTI, 1, &cause, "Failed to read the cause of rejection by the RTI."); if (cause == FEDERATION_ID_DOES_NOT_MATCH || cause == WRONG_SERVER) { lf_print_warning("Connected to the wrong RTI. Will try again"); continue; @@ -1942,7 +1917,7 @@ void lf_connect_to_rti(const char* hostname, int port) { // about connections between this federate and other federates // where messages are routed through the RTI. // @see MSG_TYPE_NEIGHBOR_STRUCTURE in net_common.h - lf_send_neighbor_structure_to_RTI(_fed.socket_TCP_RTI); + lf_send_neighbor_structure_to_RTI(_fed.net_to_RTI); uint16_t udp_port = setup_clock_synchronization_with_rti(); @@ -1950,34 +1925,39 @@ void lf_connect_to_rti(const char* hostname, int port) { unsigned char UDP_port_number[1 + sizeof(uint16_t)]; UDP_port_number[0] = MSG_TYPE_UDP_PORT; encode_uint16(udp_port, &(UDP_port_number[1])); - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, 1 + sizeof(uint16_t), UDP_port_number, NULL, - "Failed to send the UDP port number to the RTI."); + write_to_net_fail_on_error(_fed.net_to_RTI, 1 + sizeof(uint16_t), UDP_port_number, NULL, + "Failed to send the UDP port number to the RTI."); } void lf_create_server(int specified_port) { assert(specified_port <= UINT16_MAX && specified_port >= 0); - uint16_t final_port; - if (create_server(specified_port, &_fed.server_socket, &final_port, TCP, false)) { - lf_print_error_system_failure("RTI failed to create TCP server: %s.", strerror(errno)); + + net_abstraction_t* server_net = initialize_net(); + set_my_port(server_net, specified_port); + + if (create_server(server_net, false)) { + lf_print_error_system_failure("RTI failed to create server: %s.", strerror(errno)); }; + _fed.server_net = server_net; + // Get the final server port to send to the RTI on an MSG_TYPE_ADDRESS_ADVERTISEMENT message. + int32_t server_port = get_my_port(server_net); - LF_PRINT_LOG("Server for communicating with other federates started using port %u.", final_port); - _fed.server_port = final_port; + LF_PRINT_LOG("Server for communicating with other federates started using port %d.", server_port); // Send the server port number to the RTI // on an MSG_TYPE_ADDRESS_ADVERTISEMENT message (@see net_common.h). unsigned char buffer[sizeof(int32_t) + 1]; buffer[0] = MSG_TYPE_ADDRESS_ADVERTISEMENT; - encode_int32(_fed.server_port, &(buffer[1])); + encode_int32(server_port, &(buffer[1])); // Trace the event when tracing is enabled tracepoint_federate_to_rti(send_ADR_AD, _lf_my_fed_id, NULL); - // No need for a mutex because we have the only handle on this socket. - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, sizeof(int32_t) + 1, (unsigned char*)buffer, NULL, - "Failed to send address advertisement."); + // No need for a mutex because we have the only handle on this network abstraction. + write_to_net_fail_on_error(_fed.net_to_RTI, sizeof(int32_t) + 1, (unsigned char*)buffer, NULL, + "Failed to send address advertisement."); - LF_PRINT_DEBUG("Sent port %d to the RTI.", _fed.server_port); + LF_PRINT_DEBUG("Sent port %d to the RTI.", server_port); } void lf_enqueue_port_absent_reactions(environment_t* env) { @@ -2008,21 +1988,22 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) { LF_ASSERT_NON_NULL(env_arg); size_t received_federates = 0; // Allocate memory to store thread IDs. - _fed.inbound_socket_listeners = (lf_thread_t*)calloc(_fed.number_of_inbound_p2p_connections, sizeof(lf_thread_t)); + _fed.inbound_net_listeners = (lf_thread_t*)calloc(_fed.number_of_inbound_p2p_connections, sizeof(lf_thread_t)); while (received_federates < _fed.number_of_inbound_p2p_connections && !_lf_termination_executed) { // Wait for an incoming connection request. - int socket_id = accept_socket(_fed.server_socket, _fed.socket_TCP_RTI); - if (socket_id < 0) { - lf_print_warning("Federate failed to accept the socket."); + net_abstraction_t net = accept_net(_fed.server_net, _fed.net_to_RTI); + if (net == NULL) { + lf_print_warning("Federate failed to accept the network abstraction."); return NULL; } LF_PRINT_LOG("Accepted new connection from remote federate."); size_t header_length = 1 + sizeof(uint16_t) + 1; unsigned char buffer[header_length]; - int read_failed = read_from_socket(socket_id, header_length, (unsigned char*)&buffer); + int read_failed = read_from_net(net, header_length, (unsigned char*)&buffer); if (read_failed || buffer[0] != MSG_TYPE_P2P_SENDING_FED_ID) { - lf_print_warning("Federate received invalid first message on P2P socket. Closing socket."); + lf_print_warning( + "Federate received invalid first message on P2P network abstraction. Closing network abstraction."); if (read_failed == 0) { // Wrong message received. unsigned char response[2]; @@ -2031,19 +2012,19 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) { // Trace the event when tracing is enabled tracepoint_federate_to_federate(send_REJECT, _lf_my_fed_id, -3, NULL); // Ignore errors on this response. - write_to_socket(socket_id, 2, response); + write_to_net(net, 2, response); } - shutdown_socket(&socket_id, false); + shutdown_net(net, false); continue; } // Get the federation ID and check it. unsigned char federation_id_length = buffer[header_length - 1]; char remote_federation_id[federation_id_length]; - read_failed = read_from_socket(socket_id, federation_id_length, (unsigned char*)remote_federation_id); + read_failed = read_from_net(net, federation_id_length, (unsigned char*)remote_federation_id); if (read_failed || (strncmp(federation_metadata.federation_id, remote_federation_id, strnlen(federation_metadata.federation_id, 255)) != 0)) { - lf_print_warning("Received invalid federation ID. Closing socket."); + lf_print_warning("Received invalid federation ID. Closing network abstraction."); if (read_failed == 0) { unsigned char response[2]; response[0] = MSG_TYPE_REJECT; @@ -2051,9 +2032,9 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) { // Trace the event when tracing is enabled tracepoint_federate_to_federate(send_REJECT, _lf_my_fed_id, -3, NULL); // Ignore errors on this response. - write_to_socket(socket_id, 2, response); + write_to_net(net, 2, response); } - shutdown_socket(&socket_id, false); + shutdown_net(net, false); continue; } @@ -2064,12 +2045,12 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) { // Trace the event when tracing is enabled tracepoint_federate_to_federate(receive_FED_ID, _lf_my_fed_id, remote_fed_id, NULL); - // Once we record the socket_id here, all future calls to close() on - // the socket should be done while holding the socket_mutex, and this array - // element should be reset to -1 during that critical section. + // Once we record the network abstraction here, all future calls to close() on + // the network abstraction should be done while holding the net_mutex, and this array + // element should be reset to NULL during that critical section. // Otherwise, there can be race condition where, during termination, - // two threads attempt to simultaneously close the socket. - _fed.sockets_for_inbound_p2p_connections[remote_fed_id] = socket_id; + // two threads attempt to simultaneously close the network abstraction. + _fed.net_for_inbound_p2p_connections[remote_fed_id] = net; // Send an MSG_TYPE_ACK message. unsigned char response = MSG_TYPE_ACK; @@ -2077,22 +2058,23 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) { // Trace the event when tracing is enabled tracepoint_federate_to_federate(send_ACK, _lf_my_fed_id, remote_fed_id, NULL); - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); - write_to_socket_fail_on_error(&_fed.sockets_for_inbound_p2p_connections[remote_fed_id], 1, - (unsigned char*)&response, &lf_outbound_socket_mutex, - "Failed to write MSG_TYPE_ACK in response to federate %d.", remote_fed_id); + LF_MUTEX_LOCK(&lf_outbound_net_mutex); + write_to_net_fail_on_error(_fed.net_for_inbound_p2p_connections[remote_fed_id], 1, (unsigned char*)&response, + &lf_outbound_net_mutex, "Failed to write MSG_TYPE_ACK in response to federate %d.", + remote_fed_id); // Start a thread to listen for incoming messages from other federates. // The fed_id is a uint16_t, which we assume can be safely cast to and from void*. void* fed_id_arg = (void*)(uintptr_t)remote_fed_id; - int result = lf_thread_create(&_fed.inbound_socket_listeners[received_federates], listen_to_federates, fed_id_arg); + int result = lf_thread_create(&_fed.inbound_net_listeners[received_federates], listen_to_federates, fed_id_arg); if (result != 0) { // Failed to create a listening thread. - shutdown_socket(&_fed.sockets_for_inbound_p2p_connections[remote_fed_id], false); + shutdown_net(_fed.net_for_inbound_p2p_connections[remote_fed_id], false); + _fed.net_for_inbound_p2p_connections[remote_fed_id] = NULL; lf_print_error_and_exit("Failed to create a thread to listen for incoming physical connection. Error code: %d.", result); } - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_UNLOCK(&lf_outbound_net_mutex); received_federates++; } @@ -2194,23 +2176,23 @@ int lf_send_message(int message_type, unsigned short port, unsigned short federa const int header_length = 1 + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint32_t); // Use a mutex lock to prevent multiple threads from simultaneously sending. - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); + LF_MUTEX_LOCK(&lf_outbound_net_mutex); - int* socket = &_fed.sockets_for_outbound_p2p_connections[federate]; + net_abstraction_t net = _fed.net_for_outbound_p2p_connections[federate]; // Trace the event when tracing is enabled tracepoint_federate_to_federate(send_P2P_MSG, _lf_my_fed_id, federate, NULL); - int result = write_to_socket_close_on_error(socket, header_length, header_buffer); + int result = write_to_net_close_on_error(net, header_length, header_buffer); if (result == 0) { // Header sent successfully. Send the body. - result = write_to_socket_close_on_error(socket, length, message); + result = write_to_net_close_on_error(net, length, message); } if (result != 0) { // Message did not send. Since this is used for physical connections, this is not critical. lf_print_warning("Failed to send message to %s. Dropping the message.", next_destination_str); } - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_UNLOCK(&lf_outbound_net_mutex); return result; } @@ -2396,21 +2378,21 @@ void lf_send_port_absent_to_federate(environment_t* env, interval_t additional_d #ifdef FEDERATED_CENTRALIZED // Send the absent message through the RTI - int* socket = &_fed.socket_TCP_RTI; + net_abstraction_t net = _fed.net_to_RTI; tracepoint_federate_to_rti(send_PORT_ABS, _lf_my_fed_id, ¤t_message_intended_tag); #else // Send the absent message directly to the federate - int* socket = &_fed.sockets_for_outbound_p2p_connections[fed_ID]; + net_abstraction_t net = _fed.net_for_outbound_p2p_connections[fed_ID]; tracepoint_federate_to_federate(send_PORT_ABS, _lf_my_fed_id, fed_ID, ¤t_message_intended_tag); #endif - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); - int result = write_to_socket_close_on_error(socket, message_length, buffer); - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_LOCK(&lf_outbound_net_mutex); + int result = write_to_net_close_on_error(net, message_length, buffer); + LF_MUTEX_UNLOCK(&lf_outbound_net_mutex); if (result != 0) { // Write failed. Response depends on whether coordination is centralized. - if (socket == &_fed.socket_TCP_RTI) { + if (net == _fed.net_to_RTI) { // Centralized coordination. This is a critical error. lf_print_error_system_failure("Failed to send port absent message for port %hu to federate %hu.", port_ID, fed_ID); @@ -2429,29 +2411,29 @@ int lf_send_stop_request_to_rti(tag_t stop_tag) { stop_tag.microstep++; ENCODE_STOP_REQUEST(buffer, stop_tag.time, stop_tag.microstep); - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); + LF_MUTEX_LOCK(&lf_outbound_net_mutex); // Do not send a stop request if a stop request has been previously received from the RTI. if (!_fed.received_stop_request_from_rti) { LF_PRINT_LOG("Sending to RTI a MSG_TYPE_STOP_REQUEST message with tag " PRINTF_TAG ".", stop_tag.time - start_time, stop_tag.microstep); - if (_fed.socket_TCP_RTI < 0) { - lf_print_warning("Socket is no longer connected. Dropping message."); - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + if (_fed.net_to_RTI == NULL) { + lf_print_warning("RTI is no longer connected. Dropping message."); + LF_MUTEX_UNLOCK(&lf_outbound_net_mutex); return -1; } // Trace the event when tracing is enabled tracepoint_federate_to_rti(send_STOP_REQ, _lf_my_fed_id, &stop_tag); - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, MSG_TYPE_STOP_REQUEST_LENGTH, buffer, &lf_outbound_socket_mutex, - "Failed to send stop time " PRINTF_TIME " to the RTI.", stop_tag.time - start_time); + write_to_net_fail_on_error(_fed.net_to_RTI, MSG_TYPE_STOP_REQUEST_LENGTH, buffer, &lf_outbound_net_mutex, + "Failed to send stop time " PRINTF_TIME " to the RTI.", stop_tag.time - start_time); // Treat this sending as equivalent to having received a stop request from the RTI. _fed.received_stop_request_from_rti = true; - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_UNLOCK(&lf_outbound_net_mutex); return 0; } else { - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_UNLOCK(&lf_outbound_net_mutex); return 1; } } @@ -2504,14 +2486,14 @@ int lf_send_tagged_message(environment_t* env, interval_t additional_delay, int current_message_intended_tag.microstep, next_destination_str); // Use a mutex lock to prevent multiple threads from simultaneously sending. - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); + LF_MUTEX_LOCK(&lf_outbound_net_mutex); - int* socket; + net_abstraction_t net; if (message_type == MSG_TYPE_P2P_TAGGED_MESSAGE) { - socket = &_fed.sockets_for_outbound_p2p_connections[federate]; + net = _fed.net_for_outbound_p2p_connections[federate]; tracepoint_federate_to_federate(send_P2P_TAGGED_MSG, _lf_my_fed_id, federate, ¤t_message_intended_tag); } else { - socket = &_fed.socket_TCP_RTI; + net = _fed.net_to_RTI; tracepoint_federate_to_rti(send_TAGGED_MSG, _lf_my_fed_id, ¤t_message_intended_tag); } @@ -2519,10 +2501,10 @@ int lf_send_tagged_message(environment_t* env, interval_t additional_delay, int _fed.last_DNET = current_message_intended_tag; } - int result = write_to_socket_close_on_error(socket, header_length, header_buffer); + int result = write_to_net_close_on_error(net, header_length, header_buffer); if (result == 0) { // Header sent successfully. Send the body. - result = write_to_socket_close_on_error(socket, length, message); + result = write_to_net_close_on_error(net, length, message); } if (result != 0) { // Message did not send. Handling depends on message type. @@ -2533,7 +2515,7 @@ int lf_send_tagged_message(environment_t* env, interval_t additional_delay, int next_destination_str, errno, strerror(errno)); } } - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_UNLOCK(&lf_outbound_net_mutex); return result; } @@ -2575,11 +2557,11 @@ void lf_synchronize_with_other_federates(void) { start_time = get_start_time_from_rti(lf_time_physical()); lf_tracing_set_start_time(start_time); - // Start a thread to listen for incoming TCP messages from the RTI. + // Start a thread to listen for incoming messages from the RTI. // @note Up until this point, the federate has been listening for messages // from the RTI in a sequential manner in the main thread. From now on, a // separate thread is created to allow for asynchronous communication. - lf_thread_create(&_fed.RTI_socket_listener, listen_to_rti_TCP, NULL); + lf_thread_create(&_fed.RTI_net_listener, listen_to_rti_net, NULL); lf_thread_t thread_id; if (create_clock_sync_thread(&thread_id)) { lf_print_warning("Failed to create thread to handle clock synchronization."); diff --git a/core/federated/network/CMakeLists.txt b/core/federated/network/CMakeLists.txt deleted file mode 100644 index f61d69897..000000000 --- a/core/federated/network/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(LF_NETWORK_FILES net_util.c socket_common.c) - -list(TRANSFORM LF_NETWORK_FILES PREPEND federated/network/) -list(APPEND REACTORC_SOURCES ${LF_NETWORK_FILES}) diff --git a/include/core/federated/clock-sync.h b/include/core/federated/clock-sync.h index 60aed1ca1..cd5d90068 100644 --- a/include/core/federated/clock-sync.h +++ b/include/core/federated/clock-sync.h @@ -18,6 +18,7 @@ #define CLOCK_SYNC_H #include "low_level_platform.h" +#include "net_abstraction.h" #ifndef LF_CLOCK_SYNC /** @@ -233,15 +234,15 @@ uint16_t setup_clock_synchronization_with_rti(void); * is required. * * This is a blocking function that expects - * to read a MSG_TYPE_CLOCK_SYNC_T1 from the RTI TCP socket. + * to read a MSG_TYPE_CLOCK_SYNC_T1 from the RTI network abstraction. * It will then follow the PTP protocol to synchronize the local * physical clock with the RTI. * Failing to complete this protocol is treated as a catastrophic * error that causes the federate to exit. * - * @param rti_socket_TCP Pointer to the RTI's socket + * @param rti_net Pointer to the RTI's network abstraction. */ -void synchronize_initial_physical_clock_with_rti(int* rti_socket_TCP); +void synchronize_initial_physical_clock_with_rti(net_abstraction_t rti_net); /** * @brief Handle a clock synchroninzation message T1 coming from the RTI. @@ -252,31 +253,33 @@ void synchronize_initial_physical_clock_with_rti(int* rti_socket_TCP); * It also measures the time it takes between when the method is * called and the reply has been sent. * @param buffer The buffer containing the message, including the message type. - * @param socket The socket (either _lf_rti_socket_TCP or _lf_rti_socket_UDP). + * @param net_abstraction_t The pointer to the network abstraction. * @param t2 The physical time at which the T1 message was received. + * @param socket_type The socket type (TCP or UDP). * @return 0 if T3 reply is successfully sent, -1 otherwise. */ -int handle_T1_clock_sync_message(unsigned char* buffer, int socket, instant_t t2); +int handle_T1_clock_sync_message(unsigned char* buffer, void* socket_or_net, instant_t t2, socket_type_t socket_type); /** * @brief Handle a clock synchronization message T4 coming from the RTI. * @ingroup Federated * - * If the socket is _lf_rti_socket_TCP, then assume we are in the + * If the socket_or_net is a network abstraction, then assume we are in the * initial clock synchronization phase and set the clock offset * based on the estimated clock synchronization error. - * Otherwise, if the socket is _lf_rti_socket_UDP, then this looks also for a + * Otherwise, if the socket_or_net is UDP socket, then this looks also for a * subsequent "coded probe" message on the socket. If the delay between * the T4 and the coded probe message is not as expected, then reject * this clock synchronization round. If it is not rejected, then make * an adjustment to the clock offset based on the estimated error. - * This function does not acquire the socket_mutex lock. + * This function does not acquire the net_mutex lock. * The caller should acquire it unless it is sure there is only one thread running. * @param buffer The buffer containing the message, including the message type. - * @param socket The socket (either _lf_rti_socket_TCP or _lf_rti_socket_UDP). - * @param r4 The physical time at which this T4 message was received. + * @param net_abstraction_t The pointer to the network abstraction. + * @param r4 The physical time at which this T4 message was received.\ + * @param socket_type The socket type (TCP or UDP). */ -void handle_T4_clock_sync_message(unsigned char* buffer, int socket, instant_t r4); +void handle_T4_clock_sync_message(unsigned char* buffer, void* socket_or_net, instant_t r4, socket_type_t socket_type); /** * @brief Create the thread responsible for handling clock synchronization diff --git a/include/core/federated/federate.h b/include/core/federated/federate.h index 07f77a11b..e516e2a74 100644 --- a/include/core/federated/federate.h +++ b/include/core/federated/federate.h @@ -24,7 +24,7 @@ #include "lf_types.h" #include "environment.h" #include "low_level_platform.h" -#include "socket_common.h" +#include "net_abstraction.h" #ifndef ADVANCE_MESSAGE_INTERVAL #define ADVANCE_MESSAGE_INTERVAL MSEC(10) @@ -39,16 +39,16 @@ */ typedef struct federate_instance_t { /** - * The TCP socket descriptor for this federate to communicate with the RTI. + * The network abstraction for this federate to communicate with the RTI. * This is set by lf_connect_to_rti(), which must be called before other * functions that communicate with the rti are called. */ - int socket_TCP_RTI; + net_abstraction_t net_to_RTI; /** - * Thread listening for incoming TCP messages from the RTI. + * Thread listening for incoming messages from the RTI. */ - lf_thread_t RTI_socket_listener; + lf_thread_t RTI_net_listener; /** * Number of inbound physical connections to the federate. @@ -62,7 +62,7 @@ typedef struct federate_instance_t { * This is NULL if there are none and otherwise has size given by * number_of_inbound_p2p_connections. */ - lf_thread_t* inbound_socket_listeners; + lf_thread_t* inbound_net_listeners; /** * Number of outbound peer-to-peer connections from the federate. @@ -72,57 +72,50 @@ typedef struct federate_instance_t { size_t number_of_outbound_p2p_connections; /** - * An array that holds the socket descriptors for inbound + * An array that holds the network abstractions for inbound * connections from each federate. The index will be the federate * ID of the remote sending federate. This is initialized at startup - * to -1 and is set to a socket ID by lf_handle_p2p_connections_from_federates() - * when the socket is opened. + * to NULL and is set to the pointer of the network abstraction by lf_connect_to_federate() + * when the network abstractions is opened. * - * @note There will not be an inbound socket unless a physical connection + * @note There will not be an inbound network abstraction unless a physical connection * or a p2p logical connection (by setting the coordination target property * to "distributed") is specified in the Lingua Franca program where this * federate is the destination. Multiple incoming p2p connections from the - * same remote federate will use the same socket. + * same remote federate will use the same network abstraction. */ - int sockets_for_inbound_p2p_connections[NUMBER_OF_FEDERATES]; + net_abstraction_t net_for_inbound_p2p_connections[NUMBER_OF_FEDERATES]; /** - * An array that holds the socket descriptors for outbound direct + * An array that holds the network abstractions for outbound direct * connections to each remote federate. The index will be the federate * ID of the remote receiving federate. This is initialized at startup - * to -1 and is set to a socket ID by lf_connect_to_federate() - * when the socket is opened. + * to NULL and is set to the pointer of the network abstraction by lf_connect_to_federate() + * when the network abstractions is opened. * - * @note This federate will not open an outbound socket unless a physical + * @note This federate will not open an outbound network abstractions unless a physical * connection or a p2p logical connection (by setting the coordination target * property to "distributed") is specified in the Lingua Franca * program where this federate acts as the source. Multiple outgoing p2p - * connections to the same remote federate will use the same socket. + * connections to the same remote federate will use the same network abstractions. */ - int sockets_for_outbound_p2p_connections[NUMBER_OF_FEDERATES]; + net_abstraction_t net_for_outbound_p2p_connections[NUMBER_OF_FEDERATES]; /** - * Thread ID for a thread that accepts sockets and then supervises - * listening to those sockets for incoming P2P (physical) connections. + * Thread ID for a thread that accepts network abstractions and then supervises + * listening to those network abstractions for incoming P2P (physical) connections. */ lf_thread_t inbound_p2p_handling_thread_id; /** - * A socket descriptor for the socket server of the federate. + * A network abstraction for the server of the federate. * This is assigned in lf_create_server(). - * This socket is used to listen to incoming physical connections from + * This network abstraction is used to listen to incoming physical connections from * remote federates. Once an incoming connection is accepted, the - * opened socket will be stored in - * federate_sockets_for_inbound_p2p_connections. + * opened network abstraction will be stored in + * federate_net_for_inbound_p2p_connections. */ - int server_socket; - - /** - * The port used for the server socket to listen for messages from other federates. - * The federate informs the RTI of this port once it has created its socket server by - * sending an ADDRESS_AD message (@see rti.h). - */ - int server_port; + net_abstraction_t server_net; /** * Most recent tag advance grant (TAG) received from the RTI, or NEVER if none @@ -241,10 +234,10 @@ typedef enum parse_rti_code_t { SUCCESS, INVALID_PORT, INVALID_HOST, INVALID_USE // Global variables /** - * @brief Mutex lock held while performing outbound socket write and close operations. + * @brief Mutex lock held while performing outbound network abstraction write and close operations. * @ingroup Federated */ -extern lf_mutex_t lf_outbound_socket_mutex; +extern lf_mutex_t lf_outbound_net_mutex; /** * @brief Condition variable for blocking on unkonwn federate input ports. @@ -263,10 +256,10 @@ extern lf_cond_t lf_port_status_changed; * to send messages directly to the specified federate. * This function first sends an MSG_TYPE_ADDRESS_QUERY message to the RTI to obtain * the IP address and port number of the specified federate. It then attempts - * to establish a socket connection to the specified federate. + * to establish a network abstraction connection to the specified federate. * If this fails, the program exits. If it succeeds, it sets element [id] of - * the _fed.sockets_for_outbound_p2p_connections global array to - * refer to the socket for communicating directly with the federate. + * the _fed.net_for_outbound_p2p_connections global array to + * refer to the network abstraction for communicating directly with the federate. * * @param remote_federate_id The ID of the remote federate. */ @@ -276,12 +269,12 @@ void lf_connect_to_federate(uint16_t remote_federate_id); * @brief Connect to the RTI at the specified host and port. * @ingroup Federated * - * This will return the socket descriptor for the connection. + * This will return the network abstraction for the connection. * If port_number is 0, then start at DEFAULT_PORT and increment * the port number on each attempt. If an attempt fails, wait CONNECT_RETRY_INTERVAL * and try again. If it fails after CONNECT_TIMEOUT, the program exits. - * If it succeeds, it sets the _fed.socket_TCP_RTI global variable to refer to - * the socket for communicating with the RTI. + * If it succeeds, it sets the _fed.net_to_RTI global variable to refer to + * the network abstraction for communicating with the RTI. * * @param hostname A hostname, such as "localhost". * @param port_number A port number or 0 to start with the default. @@ -293,8 +286,8 @@ void lf_connect_to_rti(const char* hostname, int port_number); * @ingroup Federated * * Such connections are used for physical connections or any connection if using - * decentralized coordination. This function only handles the creation of the server socket. - * The bound port for the server socket is then sent to the RTI by sending an + * decentralized coordination. This function only handles the creation of the server network abstraction. + * The bound port for the server network abstraction is then sent to the RTI by sending an * MSG_TYPE_ADDRESS_ADVERTISEMENT message (@see net_common.h). * This function expects no response from the RTI. * @@ -324,8 +317,8 @@ void lf_enqueue_port_absent_reactions(environment_t* env); * * This thread accepts connections from federates that send messages directly * to this one (not through the RTI). This thread starts a thread for - * each accepted socket connection to read messages and, once it has opened all expected - * sockets, exits. + * each accepted network abstraction connection to read messages and, once it has opened all expected + * network abstractions, exits. * @param ignored No argument needed for this thread. */ void* lf_handle_p2p_connections_from_federates(void* ignored); @@ -369,10 +362,10 @@ void lf_reset_status_fields_on_input_port_triggers(void); * @ingroup Federated * * This function is used for physical connections - * between federates. If the socket connection to the remote federate or the RTI has been broken, + * between federates. If the connection to the remote federate or the RTI has been broken, * then this returns -1 without sending. Otherwise, it returns 0. * - * This method assumes that the caller does not hold the lf_outbound_socket_mutex lock, + * This method assumes that the caller does not hold the lf_outbound_net_mutex lock, * which it acquires to perform the send. * * @param message_type The type of the message being sent (currently only MSG_TYPE_P2P_MESSAGE). @@ -397,7 +390,7 @@ int lf_send_message(int message_type, unsigned short port, unsigned short federa * @see @ref MSG_TYPE_NEIGHBOR_STRUCTURE in @ref net_common.h * @param socket_TCP_RTI The socket descriptor for the connection to the RTI. */ -void lf_send_neighbor_structure_to_RTI(int socket_TCP_RTI); +void lf_send_neighbor_structure_to_RTI(net_abstraction_t); /** * @brief Send a next event tag (NET) signal. @@ -479,7 +472,7 @@ void lf_send_port_absent_to_federate(environment_t* env, interval_t additional_d * * The payload is the specified tag plus one microstep. If this federate has previously * received a stop request from the RTI, then do not send the message and - * return 1. Return -1 if the socket is disconnected. Otherwise, return 0. + * return 1. Return -1 if the network abstraction is disconnected. Otherwise, return 0. * @return 0 if the message is sent. */ int lf_send_stop_request_to_rti(tag_t stop_tag); @@ -492,7 +485,7 @@ int lf_send_stop_request_to_rti(tag_t stop_tag); * If the delayed tag falls after the timeout time, then the message is not sent and -1 is returned. * The caller can reuse or free the memory storing the message after this returns. * - * If the message fails to send (e.g. the socket connection is broken), then the + * If the message fails to send (e.g. the network abstraction connection is broken), then the * response depends on the message_type. For MSG_TYPE_TAGGED_MESSAGE, the message is * supposed to go via the RTI, and failure to communicate with the RTI is a critical failure. * In this case, the program will exit with an error message. If the message type is @@ -501,7 +494,7 @@ int lf_send_stop_request_to_rti(tag_t stop_tag); * to believe that there were no messages forthcoming. In this case, on failure to send * the message, this function returns -11. * - * This method assumes that the caller does not hold the lf_outbound_socket_mutex lock, + * This method assumes that the caller does not hold the lf_outbound_net_mutex lock, * which it acquires to perform the send. * * @param env The environment from which to get the current tag. @@ -564,7 +557,7 @@ void lf_stall_advance_level_federation_locked(size_t level); * @ingroup Federated * * This assumes that a connection to the RTI is already made - * and _lf_rti_socket_TCP is valid. It then sends the current logical + * and net_to_RTI is valid. It then sends the current logical * time to the RTI and waits for the RTI to respond with a specified * time. It starts a thread to listen for messages from the RTI. */ diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt index 1f7391f92..b756b406f 100644 --- a/lingua-franca-ref.txt +++ b/lingua-franca-ref.txt @@ -1 +1 @@ -master +networkdriver diff --git a/network/api/CMakeLists.txt b/network/api/CMakeLists.txt new file mode 100644 index 000000000..08aaf8105 --- /dev/null +++ b/network/api/CMakeLists.txt @@ -0,0 +1,11 @@ +add_library(lf-network-api INTERFACE) +target_include_directories(lf-network-api INTERFACE ${CMAKE_CURRENT_LIST_DIR}) +add_library(lf::network-api ALIAS lf-network-api) + +#Link necessary libraries. +target_link_libraries(lf-network-api INTERFACE lf::tag-api) +target_link_libraries(lf-network-api INTERFACE lf::low-level-platform-api) + +target_include_directories(lf-network-api INTERFACE ${CMAKE_CURRENT_LIST_DIR}/../../include/core/utils) + +target_compile_definitions(lf-network-api INTERFACE COMM_TYPE_${COMM_TYPE}) diff --git a/network/api/net_abstraction.h b/network/api/net_abstraction.h new file mode 100644 index 000000000..505d412d0 --- /dev/null +++ b/network/api/net_abstraction.h @@ -0,0 +1,290 @@ +/** + * @file net_abstraction.h + * @brief Public network abstraction API (net_abstraction) interface. + * @ingroup Network + * + * @author Dongha Kim + * @author Hokeun Kim + * + * This header defines the interface for network abstractions used by federated Lingua Franca programs. + * It abstracts network operations behind a small API that supports server/client creation, + * connection management, robust read/write with error handling, graceful shutdown, and + * querying local/peer addressing information. + */ + +#ifndef NET_ABSTRACTION_H +#define NET_ABSTRACTION_H + +#include "socket_common.h" + +typedef void* net_abstraction_t; +// net_abstraction_t +/** + * @brief Allocate and initialize a network abstraction handle. + * @ingroup Network + * + * Allocate memory for the network abstraction. + * @return net_abstraction_t Initialized network abstraction. + */ +net_abstraction_t initialize_net(); + +/** + * @brief Create a server network abstraction that will accept incoming connections. + * @ingroup Network + * + * Create a network abstraction server. This is such as a server socket which accepts connections. + * However this is only the creation of the server network abstraction. + * + * @param net_abs Server's network abstraction. + * @param increment_port_on_retry Whether to increment the port on retry if binding fails. + * @return int 0 for success, -1 for failure. + */ +int create_server(net_abstraction_t net_abs, bool increment_port_on_retry); + +/** + * @brief Accept an incoming connection on a server network abstraction. + * @ingroup Network + * + * Wait for an incoming connection request on the specified server network abstraction. + * The implementation should include three steps. + * 1. Initialize the network abstraction of the connected federate. + * 2. Wait for the incoming connection request. This should block until the connection is successfully accepted. + * 3. Save the information in the connected network abstraction, such as the address of the connected peer, for future + * querying address. + * + * @param server_chan The server network abstraction that is listening for incoming connections. + * @param rti_chan The rti's network abstraction to check if it is still open. + * @return net_abstraction_t The network abstraction for the newly accepted connection on success, or NULL on failure + */ +net_abstraction_t accept_net(net_abstraction_t server_chan, net_abstraction_t rti_chan); + +/** + * @brief Initialize a client network abstraction for connecting to a server. + * @ingroup Network + * + * Using the initialized network abstraction, create a client network abstraction ready to connect to a server. + * + * @param net_abs The initialized network abstraction. + */ +void create_client(net_abstraction_t net_abs); + +/** + * @brief Connect a client network abstraction to its configured server. + * @ingroup Network + * + * Connect to the server network abstraction. The server's connection information, + * such as the port and address should be set before calling this function. + * + * @param net_abs network abstraction to connect. + * @return 0 for success, -1 on failure, and `errno` is set to indicate the specific error. + */ +int connect_to_net(net_abstraction_t net_abs); + +/** + * @brief Read a fixed number of bytes from a network abstraction. + * @ingroup Network + * + * Read the specified number of bytes from the specified network abstraction into the specified buffer. + * If an error occurs during reading, return -1 and set errno to indicate the cause. + * If the read succeeds in reading the specified number of bytes, return 0. + * If an EOF occurs before reading the specified number of bytes, return 1. + * + * @param net_abs The network abstraction. + * @param num_bytes The number of bytes to read. + * @param buffer The buffer into which to put the bytes. + * @return 0 for success, 1 for EOF, and -1 for an error. + */ +int read_from_net(net_abstraction_t net_abs, size_t num_bytes, unsigned char* buffer); + +/** + * @brief Read bytes and close the network abstraction on error. + * @ingroup Network + * + * Uses read_from_net and closes the channel if an error occurs. + * + * @param net_abs The network abstraction. + * @param num_bytes The number of bytes to read. + * @param buffer The buffer into which to get the bytes. + * @return 0 for success, -1 for failure. + */ +int read_from_net_close_on_error(net_abstraction_t net_abs, size_t num_bytes, unsigned char* buffer); + +/** + * @brief Read bytes from a network abstraction and fail (exit) on error. + * @ingroup Network + * + * Read the specified number of bytes from the specified network abstraction into the + * specified buffer. If a disconnect or an EOF occurs during this + * reading, then if format is non-null, report an error and exit. + * If format is null, then report the error, but do not exit. + * This function takes a formatted string and additional optional arguments + * similar to printf(format, ...) that is appended to the error messages. + * + * @param net_abs The network abstraction. + * @param num_bytes The number of bytes to read. + * @param buffer The buffer into which to put the bytes. + * @param format A printf-style format string, followed by arguments to + * fill the string, or NULL to not exit with an error message. + */ +void read_from_net_fail_on_error(net_abstraction_t net_abs, size_t num_bytes, unsigned char* buffer, char* format, ...); + +/** + * @brief Write a fixed number of bytes to a network abstraction. + * @ingroup Network + * + * Write the specified number of bytes to the specified network abstraction using write_to_net + * and close the network abstraction if an error occurs. + * If an error occurs, return -1 and set errno to indicate the cause. If the write succeeds, return 0. + * This function retries until the specified number of bytes have been written or an error occurs. + * + * @param net_abs The network abstraction. + * @param num_bytes The number of bytes to write. + * @param buffer The buffer from which to get the bytes. + * @return 0 for success, -1 for failure. + */ +int write_to_net(net_abstraction_t net_abs, size_t num_bytes, unsigned char* buffer); + +/** + * @brief Write bytes to a network abstraction and close on error. + * @ingroup Network + * + * Uses write_to_net and closes the channel if an error occurs. + * + * @param net_abs The network abstraction. + * @param num_bytes The number of bytes to write. + * @param buffer The buffer from which to get the bytes. + * @return 0 for success, -1 for failure. + */ +int write_to_net_close_on_error(net_abstraction_t net_abs, size_t num_bytes, unsigned char* buffer); + +/** + * @brief Write bytes to a network abstraction and fail (exit) on error. + * @ingroup Network + * + * Write the specified number of bytes to the specified network abstraction using + * write_to_net_close_on_error and exit with an error code if an error occurs. + * If the mutex argument is non-NULL, release the mutex before exiting. If the + * format argument is non-null, then use it an any additional arguments to form + * the error message using printf conventions. Otherwise, print a generic error + * message. + * @param net_abs The network abstraction. + * @param num_bytes The number of bytes to write. + * @param buffer The buffer from which to get the bytes. + * @param mutex If non-NULL, the mutex to unlock before exiting. + * @param format A format string for error messages, followed by any number of + * fields that will be used to fill the format string as in printf, or NULL + * to print a generic error message. + */ +void write_to_net_fail_on_error(net_abstraction_t net_abs, size_t num_bytes, unsigned char* buffer, lf_mutex_t* mutex, + char* format, ...); + +/** + * @brief Check whether the network abstraction is closed. + * @ingroup Network + * + * Checks if the network abstraction is still connected to the peer. + * + * @param net_abs The network abstraction. + * @return true if closed, false if still open. + */ +bool check_net_closed(net_abstraction_t net_abs); + +/** + * @brief Gracefully shut down and close a network abstraction. + * @ingroup Network + * + * If read_before_closing is false, call shutdown() with SHUT_RDWR and then close(). If true, call shutdown() with + * SHUT_WR, then read() until EOF and discard received bytes before closing. + * + * @param net_abs The network abstraction to shut down and close. + * @param net_abs The network abstraction to shutdown and close. + * @param read_before_closing If true, read until EOF before closing the network abstraction. + * @return int Returns 0 on success, -1 on failure (errno will indicate the error). + */ +int shutdown_net(net_abstraction_t net_abs, bool read_before_closing); + +/** + * @brief Get the server port number of this network abstraction. + * @ingroup Network + * + * Get the open port number from the network abstraction. + * This is used when the federate sends a MSG_TYPE_ADDRESS_ADVERTISEMENT to the RTI, informing its port number. The RTI + * will save this port number, and send it to the other federate in a MSG_TYPE_ADDRESS_QUERY_REPLY message. + * + * @param net_abs The network abstraction. + * @return The port number of a server network abstraction. + */ +int32_t get_my_port(net_abstraction_t net_abs); + +/** + * @brief Get the connected peer's port number. + * @ingroup Network + * + * Get the port number of the connected peer. + * This is used by the RTI, when there is a request from the federate to the RTI, for the MSG_TYPE_ADDRESS_QUERY + * message. + * + * @param net_abs The network abstraction. + * @return Port number of the connected peer. + */ +int32_t get_server_port(net_abstraction_t net_abs); + +/** + * @brief Get the connected peer's IP address. + * @ingroup Network + * + * Get the IP address of the connected peer. + * + * @param net_abs The network abstraction. + * @return Pointer to the server IP address. + */ +struct in_addr* get_ip_addr(net_abstraction_t net_abs); + +/** + * @brief Get the connected peer's hostname. + * @ingroup Network + * + * Get the hostname of the connected peer. + * + * @param net_abs The network abstraction. + * @return Pointer to the server hostname. + */ +char* get_server_hostname(net_abstraction_t net_abs); + +/** + * @brief Set the user-specified port for this network abstraction. + * @ingroup Network + * + * Set the user specified port to the created network abstraction. + * + * @param net_abs The network abstraction. + * @param port The user specified port. + */ +void set_my_port(net_abstraction_t net_abs, int32_t port); + +/** + * @brief Set the target server's port number for this network abstraction. + * @ingroup Network + * + * Set server port number to the target network abstraction. + * The federate and RTI receives the port number from another + * federate MSG_TYPE_ADDRESS_ADVERTISEMENT message. + * This function is used to set the network abstraction's target server port number. + * + * @param net_abs The network abstraction. + * @param port The target server's port. + */ +void set_server_port(net_abstraction_t net_abs, int32_t port); + +/** + * @brief Set the target server's port number for this network abstraction. + * @ingroup Network + * + * Set the target server's hostname to the network abstraction. + * + * @param net_abs The network abstraction. + * @param hostname The target server's hostname. + */ +void set_server_hostname(net_abstraction_t net_abs, const char* hostname); + +#endif /* NET_ABSTRACTION_H */ diff --git a/include/core/federated/network/net_common.h b/network/api/net_common.h similarity index 96% rename from include/core/federated/network/net_common.h rename to network/api/net_common.h index 07641fdbf..8f4855fcc 100644 --- a/include/core/federated/network/net_common.h +++ b/network/api/net_common.h @@ -1,7 +1,7 @@ /** * @file net_common.h * @brief Common message types and definitions for federated Lingua Franca programs. - * @ingroup Federated + * @ingroup Network * * @author Edward A. Lee * @author Soroush Bateni @@ -185,7 +185,7 @@ /** * @brief Size of the buffer used for messages sent between federates. - * @ingroup Federated + * @ingroup Network * * This is used by both the federates and the RTI, so message lengths * should generally match. @@ -194,7 +194,7 @@ /** * @brief Time that a federate waits before asking the RTI again for the port and IP address of a federate. - * @ingroup Federated + * @ingroup Network * * The federate repeatedly sends an MSG_TYPE_ADDRESS_QUERY message after the RTI responds that it * does not know to previous such messages. This allows time for federates to start separately. @@ -203,7 +203,7 @@ /** * @brief Delay the start of all federates by this amount. - * @ingroup Federated + * @ingroup Network * * This helps ensure that the federates do not start at the same time. * Each federate has provided its current physical time to the RTI, and @@ -225,7 +225,7 @@ /** * @brief Byte identifying a rejection of the previously received message. - * @ingroup Federated + * @ingroup Network * * The reason for the rejection is included as an additional byte * (uchar) (see below for encodings of rejection reasons). @@ -234,7 +234,7 @@ /** * @brief Byte identifying an acknowledgment of the previously received message. - * @ingroup Federated + * @ingroup Network * * This message carries no payload. */ @@ -242,7 +242,7 @@ /** * @brief Byte identifying an acknowledgment of the previously received MSG_TYPE_FED_IDS message. - * @ingroup Federated + * @ingroup Network * * This message is sent by the RTI to the federate with a payload indicating the UDP port to use * for clock synchronization. The next four bytes will be the port number for the UDP server, or @@ -254,7 +254,7 @@ /** * @brief Byte identifying a message from a federate to an RTI containing * the federation ID and the federate ID. - * @ingroup Federated + * @ingroup Network * * The message contains, in this order: * * One byte equal to MSG_TYPE_FED_IDS. @@ -276,7 +276,7 @@ /** * @brief Byte identifying a message from a federate to an RTI containing * federate's 8-byte random nonce for HMAC-based authentication. - * @ingroup Federated + * @ingroup Network * * The federate sends this message to an incoming RTI when TCP connection is established * between the RTI and the federate. @@ -290,7 +290,7 @@ /** * @brief Byte identifying a message from RTI to federate as a response to the FED_NONCE * message. - * @ingroup Federated + * @ingroup Network * * The RTI sends this message to federate for HMAC-based authentication. * The message contains, in this order: @@ -307,7 +307,7 @@ /** * @brief Byte identifying a message from federate to RTI as a response to the RTI_RESPONSE * message. - * @ingroup Federated + * @ingroup Network * * The federate sends this message to RTI for HMAC-based authentication. * The message contains, in this order: @@ -321,19 +321,19 @@ /** * @brief The randomly created nonce size will be 8 bytes. - * @ingroup Federated + * @ingroup Network */ #define NONCE_LENGTH 8 /** * @brief The HMAC tag uses the SHA256 hash algorithm, creating a 32 byte length hash tag. - * @ingroup Federated + * @ingroup Network */ #define SHA256_HMAC_LENGTH 32 /** * @brief Byte identifying a timestamp message, which is 64 bits long. - * @ingroup Federated + * @ingroup Network * * Each federate sends its starting physical time as a message of this * type, and the RTI broadcasts to all the federates the starting logical @@ -343,13 +343,13 @@ /** * @brief The length of a timestamp message. - * @ingroup Federated + * @ingroup Network */ #define MSG_TYPE_TIMESTAMP_LENGTH (1 + sizeof(int64_t)) /** * @brief Byte identifying a message to forward to another federate. - * @ingroup Federated + * @ingroup Network * * The next two bytes will be the ID of the destination port. * The next two bytes are the destination federate ID. @@ -363,13 +363,13 @@ /** * @brief Byte identifying that the federate or the RTI is ending its execution. - * @ingroup Federated + * @ingroup Network */ #define MSG_TYPE_RESIGN 4 /** * @brief Byte identifying a timestamped message to forward to another federate. - * @ingroup Federated + * @ingroup Network * * The next two bytes will be the ID of the destination reactor port. * The next two bytes are the destination federate ID. @@ -387,7 +387,7 @@ /** * @brief Byte identifying a next event tag (NET) message sent from a federate in * centralized coordination. - * @ingroup Federated + * @ingroup Network * * The next eight bytes will be the timestamp. The next four bytes will be the microstep. * This message from a federate tells the RTI the tag of the earliest event on that @@ -406,7 +406,7 @@ /** * @brief Byte identifying a time advance grant (TAG) sent by the RTI to a federate * in centralized coordination. - * @ingroup Federated + * @ingroup Network * * This message is a promise by the RTI to the federate that no later message sent to the * federate will have a tag earlier than or equal to the tag carried by this TAG message. @@ -418,7 +418,7 @@ /** * @brief Byte identifying a provisional time advance grant (PTAG) sent by the RTI to a federate * in centralized coordination. - * @ingroup Federated + * @ingroup Network * * This message is a promise by the RTI to the federate that no later message sent to the * federate will have a tag earlier than the tag carried by this PTAG message. @@ -430,7 +430,7 @@ /** * @brief Byte identifying a latest tag confirmed (LTC) message sent by a federate * to the RTI. - * @ingroup Federated + * @ingroup Network * * The next eight bytes will be the timestep of the completed tag. * The next four bytes will be the microsteps of the completed tag. @@ -463,13 +463,13 @@ /** * @brief The length of a stop request message. - * @ingroup Federated + * @ingroup Network */ #define MSG_TYPE_STOP_REQUEST_LENGTH (1 + sizeof(instant_t) + sizeof(microstep_t)) /** * @brief Encode a stop request message. - * @ingroup Federated + * @ingroup Network * * @param buffer The buffer to encode the message into. * @param time The time at which the federates will stop. @@ -494,13 +494,13 @@ /** * @brief The length of a stop request reply message. - * @ingroup Federated + * @ingroup Network */ #define MSG_TYPE_STOP_REQUEST_REPLY_LENGTH (1 + sizeof(instant_t) + sizeof(microstep_t)) /** * @brief Encode a stop request reply message. - * @ingroup Federated + * @ingroup Network * * @param buffer The buffer to encode the message into. * @param time The time at which the federates will stop. @@ -516,7 +516,7 @@ /** * @brief Byte sent by the RTI indicating that the stop request from some federate * has been granted. - * @ingroup Federated + * @ingroup Network * * The payload is the tag at which all federates have agreed that they can stop. * The next 8 bytes will be the time at which the federates will stop. @@ -526,13 +526,13 @@ /** * @brief The length of a stop granted message. - * @ingroup Federated + * @ingroup Network */ #define MSG_TYPE_STOP_GRANTED_LENGTH (1 + sizeof(instant_t) + sizeof(microstep_t)) /** * @brief Encode a stop granted message. - * @ingroup Federated + * @ingroup Network * * @param buffer The buffer to encode the message into. * @param time The time at which the federates will stop. @@ -550,7 +550,7 @@ /** * @brief Byte identifying a address query message, sent by a federate to RTI * to ask for another federate's address and port number. - * @ingroup Federated + * @ingroup Network * * The next two bytes are the other federate's ID. */ @@ -559,7 +559,7 @@ /** * @brief Byte identifying a address query message reply, sent by a RTI to a federate * to reply with a remote federate's address and port number. - * @ingroup Federated + * @ingroup Network * * The reply from the RTI will be a port number (an int32_t), which is -1 * if the RTI does not know yet (it has not received MSG_TYPE_ADDRESS_ADVERTISEMENT from @@ -573,7 +573,7 @@ /** * @brief Byte identifying a message advertising the port for the TCP connection server * of a federate. - * @ingroup Federated + * @ingroup Network * * This is utilized in decentralized coordination as well as for physical * connections in centralized coordination. @@ -586,7 +586,7 @@ /** * @brief Byte identifying a first message that is sent by a federate directly to another federate * after establishing a socket connection to send messages directly to the federate. - * @ingroup Federated + * @ingroup Network * * This * first message contains two bytes identifying the sending federate (its ID), a byte @@ -599,7 +599,7 @@ /** * @brief Byte identifying a message to send directly to another federate. - * @ingroup Federated + * @ingroup Network * * The next two bytes will be the ID of the destination port. * The next two bytes are the destination federate ID. This is checked against @@ -611,7 +611,7 @@ /** * @brief Byte identifying a timestamped message to send directly to another federate. - * @ingroup Federated + * @ingroup Network * * This is a variant of @see MSG_TYPE_TAGGED_MESSAGE that is used in P2P connections between * federates. Having a separate message type for P2P connections between federates @@ -631,7 +631,7 @@ //////////////////////////////////////////////// /** * @brief Physical clock synchronization messages according to PTP. - * @ingroup Federated + * @ingroup Network * * The next 8 bytes will be a timestamp sent according to PTP. */ @@ -639,7 +639,7 @@ /** * @brief Prompt the master to send a T4. - * @ingroup Federated + * @ingroup Network * * The next four bytes will be the sending federate's id. */ @@ -647,7 +647,7 @@ /** * @brief Physical clock synchronization message according to PTP. - * @ingroup Federated + * @ingroup Network * * The next 8 bytes will be a timestamp sent according to PTP. */ @@ -655,7 +655,7 @@ /** * @brief Coded probe message. - * @ingroup Federated + * @ingroup Network * * This messages is sent by the server (master) * right after MSG_TYPE_CLOCK_SYNC_T4(t1) with a new physical clock snapshot t2. @@ -670,7 +670,7 @@ /** * @brief A port absent message, informing the receiver that a given port * will not have event for the current logical time. - * @ingroup Federated + * @ingroup Network * * The next 2 bytes is the port id. * The next 2 bytes will be the federate id of the destination federate. @@ -684,7 +684,7 @@ /** * @brief A message that informs the RTI about connections between this federate and * other federates where messages are routed through the RTI. - * @ingroup Federated + * @ingroup Network * * Currently, this only includes logical connections when the coordination is centralized. * This information is needed for the RTI to perform the centralized coordination. @@ -712,20 +712,20 @@ /** * @brief The size of the header of a neighbor structure message. - * @ingroup Federated + * @ingroup Network */ #define MSG_TYPE_NEIGHBOR_STRUCTURE_HEADER_SIZE 9 /** * @brief Byte identifying that the federate or the RTI has failed. - * @ingroup Federated + * @ingroup Network */ #define MSG_TYPE_FAILED 25 /** * @brief Byte identifying a downstream next event tag (DNET) message sent * from the RTI in centralized coordination. - * @ingroup Federated + * @ingroup Network * * The next eight bytes will be the timestamp. * The next four bytes will be the microstep. @@ -742,49 +742,49 @@ /** * @brief Code sent with a @ref MSG_TYPE_REJECT message indicating that the * federation ID does not match. - * @ingroup Federated + * @ingroup Network */ #define FEDERATION_ID_DOES_NOT_MATCH 1 /** * @brief Code sent with a @ref MSG_TYPE_REJECT message indicating that the * federate ID is already in use. - * @ingroup Federated + * @ingroup Network */ #define FEDERATE_ID_IN_USE 2 /** * @brief Code sent with a @ref MSG_TYPE_REJECT message indicating that the * federate ID is out of range. - * @ingroup Federated + * @ingroup Network */ #define FEDERATE_ID_OUT_OF_RANGE 3 /** * @brief Code sent with a @ref MSG_TYPE_REJECT message indicating that the * incoming message is not expected. - * @ingroup Federated + * @ingroup Network */ #define UNEXPECTED_MESSAGE 4 /** * @brief Code sent with a @ref MSG_TYPE_REJECT message indicating that the * connected to the wrong server. - * @ingroup Federated + * @ingroup Network */ #define WRONG_SERVER 5 /** * @brief Code sent with a @ref MSG_TYPE_REJECT message indicating that the * HMAC authentication failed. - * @ingroup Federated + * @ingroup Network */ #define HMAC_DOES_NOT_MATCH 6 /** * @brief Code sent with a @ref MSG_TYPE_REJECT message indicating that the * RTI was not executed using the -a or --auth option. - * @ingroup Federated + * @ingroup Network */ #define RTI_NOT_EXECUTED_WITH_AUTH 7 diff --git a/include/core/federated/network/net_util.h b/network/api/net_util.h similarity index 94% rename from include/core/federated/network/net_util.h rename to network/api/net_util.h index ebc5e3ba7..bdf7bccf0 100644 --- a/include/core/federated/network/net_util.h +++ b/network/api/net_util.h @@ -1,7 +1,7 @@ /** * @file net_util.h * @brief Network utility functions for Lingua Franca programs. - * @ingroup Federated + * @ingroup Network * * @author Edward A. Lee * @author Soroush Bateni @@ -30,22 +30,18 @@ #include "low_level_platform.h" #include "tag.h" -#ifdef FEDERATED -#include "socket_common.h" -#endif - #define HOST_LITTLE_ENDIAN 1 #define HOST_BIG_ENDIAN 2 /** * @brief Return true (1) if the host is big endian. Otherwise, return false. - * @ingroup Federated + * @ingroup Network */ int host_is_big_endian(void); /** * @brief Write the specified data as a sequence of bytes starting at the specified address. - * @ingroup Federated + * @ingroup Network * * This encodes the data in little-endian order (lowest order byte first). * @param data The data to write. @@ -55,7 +51,7 @@ void encode_int64(int64_t data, unsigned char* buffer); /** * @brief Write the specified data as a sequence of bytes starting at the specified address. - * @ingroup Federated + * @ingroup Network * * This encodes the data in little-endian order (lowest order byte first). * This works for int32_t. @@ -66,7 +62,7 @@ void encode_int32(int32_t data, unsigned char* buffer); /** * @brief Write the specified data as a sequence of bytes starting at the specified address. - * @ingroup Federated + * @ingroup Network * * This encodes the data in little-endian order (lowest order byte first). * This works for uint32_t. @@ -77,7 +73,7 @@ void encode_uint32(uint32_t data, unsigned char* buffer); /** * @brief Write the specified data as a sequence of bytes starting at the specified address. - * @ingroup Federated + * @ingroup Network * * This encodes the data in little-endian order (lowest order byte first). * @param data The data to write. @@ -87,7 +83,7 @@ void encode_uint16(uint16_t data, unsigned char* buffer); /** * @brief If this host is little endian, then reverse the order of the bytes of the argument. - * @ingroup Federated + * @ingroup Network * * Otherwise, return the argument unchanged. * This can be used to convert the argument to network order (big endian) and then back again. @@ -101,7 +97,7 @@ int32_t swap_bytes_if_big_endian_int32(int32_t src); /** * @brief If this host is little endian, then reverse the order of the bytes of the argument. - * @ingroup Federated + * @ingroup Network * * Otherwise, return the argument unchanged. * This can be used to convert the argument to network order (big endian) and then back again. @@ -113,7 +109,7 @@ int64_t swap_bytes_if_big_endian_int64(int64_t src); /** * @brief If this host is little endian, then reverse the order of the bytes of the argument. - * @ingroup Federated + * @ingroup Network * * Otherwise, return the argument unchanged. * This can be used to convert the argument to network order (big endian) and then back again. @@ -127,7 +123,7 @@ uint16_t swap_bytes_if_big_endian_uint16(uint16_t src); /** * @brief This will swap the order of the bytes if this machine is big endian. - * @ingroup Federated + * @ingroup Network * * @param bytes The address of the start of the sequence of bytes. */ @@ -135,7 +131,7 @@ int32_t extract_int32(unsigned char* bytes); /** * @brief This will swap the order of the bytes if this machine is big endian. - * @ingroup Federated + * @ingroup Network * * @param bytes The address of the start of the sequence of bytes. */ @@ -143,7 +139,7 @@ int64_t extract_int64(unsigned char* bytes); /** * @brief Extract an uint16_t from the specified byte sequence. - * @ingroup Federated + * @ingroup Network * * This will swap the order of the bytes if this machine is big endian. * @param bytes The address of the start of the sequence of bytes. @@ -154,7 +150,7 @@ uint16_t extract_uint16(unsigned char* bytes); /** * @brief Extract the core header information that all messages between federates share. - * @ingroup Federated + * @ingroup Network * * The core header information is two bytes with the ID of the destination port, * two bytes with the ID of the destination federate, and four bytes with the length of the message. @@ -168,7 +164,7 @@ void extract_header(unsigned char* buffer, uint16_t* port_id, uint16_t* federate /** * @brief Extract the timed header information for timed messages between federates. - * @ingroup Federated + * @ingroup Network * * This is two bytes with the ID of the destination port, two bytes with the ID of the destination * federate, four bytes with the length of the message, eight bytes with a timestamp, and four bytes with a microstep. @@ -182,7 +178,7 @@ void extract_timed_header(unsigned char* buffer, uint16_t* port_id, uint16_t* fe /** * @brief Extract tag information from buffer. - * @ingroup Federated + * @ingroup Network * * The tag is transmitted as a 64-bit (8 byte) signed integer for time and a 32-bit (4 byte) unsigned integer for * microstep. @@ -193,7 +189,7 @@ tag_t extract_tag(unsigned char* buffer); /** * @brief Encode tag information into buffer. - * @ingroup Federated + * @ingroup Network * * Buffer must have been allocated externally. * @param buffer The buffer to encode into. @@ -203,7 +199,7 @@ void encode_tag(unsigned char* buffer, tag_t tag); /** * @brief A helper struct for passing rti_addr information between lf_parse_rti_addr and extract_rti_addr_info - * @ingroup Federated + * @ingroup Network * */ typedef struct rti_addr_info_t { @@ -217,7 +213,7 @@ typedef struct rti_addr_info_t { /** * @brief Check whether str matches regex. - * @ingroup Federated + * @ingroup Network * * @param str The string to check. * @param regex The regex to check against. @@ -227,7 +223,7 @@ bool match_regex(const char* str, char* regex); /** * @brief Check whether port is valid. - * @ingroup Federated + * @ingroup Network * * @param port The port to check. * @return true if valid, false otherwise. @@ -236,7 +232,7 @@ bool validate_port(char* port); /** * @brief Check whether host is valid. - * @ingroup Federated + * @ingroup Network * * @param host The host to check. * @return true if valid, false otherwise. @@ -245,7 +241,7 @@ bool validate_host(const char* host); /** * @brief Check whether user is valid. - * @ingroup Federated + * @ingroup Network * * @param user The user to check. * @return true if valid, false otherwise. @@ -254,7 +250,7 @@ bool validate_user(const char* user); /** * @brief Extract one match group from the rti_addr regex . - * @ingroup Federated + * @ingroup Network * * @param rti_addr The rti_addr to extract from. * @param dest The destination to store the match group. @@ -269,7 +265,7 @@ bool extract_match_group(const char* rti_addr, char* dest, regmatch_t group, siz /** * @brief Extract match groups from the rti_addr regex. - * @ingroup Federated + * @ingroup Network * * @param rti_addr The rti_addr to extract from. * @param rti_addr_strs The array of rti_addr strings to store the match groups. @@ -286,7 +282,7 @@ bool extract_match_groups(const char* rti_addr, char** rti_addr_strs, bool** rti /** * @brief Extract the host, port and user from rti_addr. - * @ingroup Federated + * @ingroup Network * * @param rti_addr The rti_addr to extract from. * @param rti_addr_info The rti_addr_info into which to store the extracted information. diff --git a/include/core/federated/network/socket_common.h b/network/api/socket_common.h similarity index 83% rename from include/core/federated/network/socket_common.h rename to network/api/socket_common.h index 2d5b1a952..c14633f1d 100644 --- a/include/core/federated/network/socket_common.h +++ b/network/api/socket_common.h @@ -1,25 +1,30 @@ /** * @file socket_common.h * @brief Common socket operations and utilities for federated Lingua Franca programs. - * @ingroup Federated + * @ingroup Network * * @author Edward A. Lee * @author Soroush Bateni * @author Peter Donovan + * @author Dongha Kim * * This file provides common socket operations and utilities used in federated Lingua Franca programs. * It includes functions for creating and managing TCP/UDP sockets, handling connections, * and performing read/write operations with proper error handling and retry mechanisms. * The file also defines various constants for timeouts, retry intervals, and port configurations. */ + #ifndef SOCKET_COMMON_H #define SOCKET_COMMON_H -#include "low_level_platform.h" +#include "low_level_platform.h" // lf_mutex_t +#include +#include +#include /** * @brief The number of federates. - * @ingroup Federated + * @ingroup Network * * This defaults to 1. */ @@ -29,7 +34,7 @@ /** * @brief The amount of time to wait after a failed socket read or write before trying again. - * @ingroup Federated + * @ingroup Network * * This defaults to 100 ms. */ @@ -37,7 +42,7 @@ /** * @brief The timeout time in ns for TCP operations. - * @ingroup Federated + * @ingroup Network * * Default value is 10 secs. */ @@ -45,7 +50,7 @@ /** * @brief The timeout time in ns for UDP operations. - * @ingroup Federated + * @ingroup Network * * Default value is 1 sec. */ @@ -53,13 +58,13 @@ /** * @brief Time between a federate's attempts to connect to the RTI. - * @ingroup Federated + * @ingroup Network */ #define CONNECT_RETRY_INTERVAL MSEC(500) /** * @brief Bound on the number of retries to connect to the RTI. - * @ingroup Federated + * @ingroup Network * * A federate will retry every CONNECT_RETRY_INTERVAL nanoseconds until * CONNECTION_TIMEOUT expires. @@ -68,7 +73,7 @@ /** * @brief Maximum number of port addresses that a federate will try to connect to the RTI on. - * @ingroup Federated + * @ingroup Network * * If you are using automatic ports begining at DEFAULT_PORT, this puts an upper bound * on the number of RTIs that can be running on the same host. @@ -77,7 +82,7 @@ /** * @brief Time to wait before re-attempting to bind to a port. - * @ingroup Federated + * @ingroup Network * * When a process closes, the network stack typically waits between 30 and 120 * seconds before releasing the port. This is to allow for delayed packets so @@ -88,13 +93,13 @@ /** * @brief Number of attempts to bind to a port before giving up. - * @ingroup Federated + * @ingroup Network */ #define PORT_BIND_RETRY_LIMIT 60 /** * @brief Default port number for the RTI. - * @ingroup Federated + * @ingroup Network * * Unless a specific port has been specified by the LF program in the "at" * for the RTI or on the command line, when the RTI starts up, it will attempt @@ -102,32 +107,55 @@ */ #define DEFAULT_PORT 15045u +/** + * Default port number for the RTI's clock server. + */ +#define DEFAULT_UDP_PORT 15061u + /** * @brief Byte identifying that the federate or the RTI has failed. - * @ingroup Federated + * @ingroup Network */ #define MSG_TYPE_FAILED 25 /** * @brief Type of socket. - * @ingroup Federated + * @ingroup Network */ typedef enum socket_type_t { TCP, UDP } socket_type_t; +typedef struct socket_priv_t { + /** @brief The TCP socket descriptor for the socket server. */ + int socket_descriptor; + /** @brief The final port number that the TCP socket server ends up using. */ + uint16_t port; + /** @brief The desired port specified by the user on the command line. */ + uint16_t user_specified_port; + /** @brief Human-readable IP address of the federate's socket server. */ + char server_hostname[INET_ADDRSTRLEN]; + /** @brief Port number of the socket server of the federate. The port number will be -1 if there is no server or if + * the RTI has not been informed of the port number. */ + int32_t server_port; + /** @brief Information about the IP address of the socket server of the federate. */ + struct in_addr server_ip_addr; + /** @brief The UDP address for the federate. */ + struct sockaddr_in UDP_addr; +} socket_priv_t; + /** * @brief Create an IPv4 TCP socket with Nagle's algorithm disabled. - * @ingroup Federated + * @ingroup Network * * This uses TCP_NODELAY and Delayed ACKs disabled with TCP_QUICKACK. * It exits application on any error. * * @return The socket ID (a file descriptor). */ -int create_real_time_tcp_socket_errexit(); +int create_real_time_tcp_socket_errexit(void); /** * @brief Create a TCP server that listens for socket connections. - * @ingroup Federated + * @ingroup Network * * If the specified port number is greater than zero, this function will attempt to acquire that port. * If the specified port number is zero, and the increment_port_on_retry is true, it will attempt to acquire @@ -146,12 +174,12 @@ int create_real_time_tcp_socket_errexit(); * @param increment_port_on_retry Boolean to retry port increment. * @return 0 for success, -1 for failure. */ -int create_server(uint16_t port, int* final_socket, uint16_t* final_port, socket_type_t sock_type, - bool increment_port_on_retry); +int create_socket_server(uint16_t port, int* final_socket, uint16_t* final_port, socket_type_t sock_type, + bool increment_port_on_retry); /** * @brief Wait for an incoming connection request on the specified server socket. - * @ingroup Federated + * @ingroup Network * * This blocks until a connection is successfully accepted. If an error occurs that is not * temporary (e.g., `EAGAIN` or `EWOULDBLOCK`), it reports the error and exits. Temporary @@ -167,12 +195,11 @@ int create_server(uint16_t port, int* final_socket, uint16_t* final_port, socket * @return The file descriptor for the newly accepted socket on success, or -1 on failure * (with an appropriate error message printed). */ - int accept_socket(int socket, int rti_socket); /** * @brief Attempt to establish a TCP connection to the specified hostname and port. - * @ingroup Federated + * @ingroup Network * * Attempt to establish a TCP connection to the specified hostname * and port. This function uses `getaddrinfo` to resolve the hostname and retries the connection @@ -189,7 +216,7 @@ int connect_to_socket(int sock, const char* hostname, int port); /** * @brief Read the specified number of bytes from the specified socket into the specified buffer. - * @ingroup Federated + * @ingroup Network * * If an error occurs during this reading, return -1 and set errno to indicate * the cause of the error. If the read succeeds in reading the specified number of bytes, @@ -207,7 +234,7 @@ int read_from_socket(int socket, size_t num_bytes, unsigned char* buffer); /** * @brief Read the specified number of bytes from the specified socket into the specified buffer. - * @ingroup Federated + * @ingroup Network * * This uses @ref read_from_socket, but if a failure occurs, it closes the socket using * @ref shutdown_socket and returns -1. Otherwise, it returns 0. @@ -221,7 +248,7 @@ int read_from_socket_close_on_error(int* socket, size_t num_bytes, unsigned char /** * @brief Read the specified number of bytes from the specified socket into the specified * buffer and close the socket if an error occurs. - * @ingroup Federated + * @ingroup Network * * If a disconnect or an EOF occurs during this reading, then if format is non-null, * report an error and exit. @@ -239,7 +266,7 @@ void read_from_socket_fail_on_error(int* socket, size_t num_bytes, unsigned char /** * @brief Without blocking, peek at the specified socket. - * @ingroup Federated + * @ingroup Network * * If there is anything on the queue, put its first byte at the specified address and return 1. * If there is nothing on the queue, return 0, and if an error occurs, return -1. @@ -249,9 +276,31 @@ void read_from_socket_fail_on_error(int* socket, size_t num_bytes, unsigned char */ ssize_t peek_from_socket(int socket, unsigned char* result); +/** + * @brief Check if the socket is closed. + * @ingroup Network + * + * Return true if either the socket to the RTI is broken or the socket is + * alive and the first unread byte on the socket's queue is MSG_TYPE_FAILED. + * @param socket Socket to check. + * @return True if closed, false if still connected. + */ +bool check_socket_closed(int socket); +/** + * @brief Get the connected peer address. + * @ingroup Network + * + * Get the connected peer name from the connected socket. + * Set it to the server_ip_addr. Also, set server_hostname if LOG_LEVEL is higher than LOG_LEVEL_DEBUG. + * + * @param priv The socket_priv struct. + * @return 0 for success, -1 for failure. + */ +int get_peer_address(socket_priv_t* priv); + /** * @brief Write the specified number of bytes to the specified socket from the specified buffer. - * @ingroup Federated + * @ingroup Network * * If an error occurs, return -1 and set errno to indicate the cause of the error. * If the write succeeds, return 0. @@ -269,7 +318,7 @@ int write_to_socket(int socket, size_t num_bytes, unsigned char* buffer); /** * @brief Write the specified number of bytes to the specified socket. - * @ingroup Federated + * @ingroup Network * * This uses @ref write_to_socket and closes the socket if an error occurs. * If an error occurs, this will change the socket ID pointed to by the first argument to -1 and will return -1. @@ -283,7 +332,7 @@ int write_to_socket_close_on_error(int* socket, size_t num_bytes, unsigned char* /** * @brief Write the specified number of bytes to the specified socket. - * @ingroup Federated + * @ingroup Network * * This uses @ref write_to_socket_close_on_error and exits with an error code if an error occurs. * If the mutex argument is non-NULL, release the mutex before exiting. @@ -304,7 +353,7 @@ void write_to_socket_fail_on_error(int* socket, size_t num_bytes, unsigned char* /** * @brief Initialize shutdown mutex. - * @ingroup Federated + * @ingroup Network * * This is used to synchronize the shutdown of the federate. */ @@ -312,7 +361,7 @@ void init_shutdown_mutex(void); /** * @brief Shutdown and close the socket. - * @ingroup Federated + * @ingroup Network * * If `read_before_closing` is false, this calls `shutdown` with `SHUT_RDWR`, shutting down both directions. * If this fails, then it calls `close`. diff --git a/network/impl/CMakeLists.txt b/network/impl/CMakeLists.txt new file mode 100644 index 000000000..225edf3d5 --- /dev/null +++ b/network/impl/CMakeLists.txt @@ -0,0 +1,25 @@ +set(LF_ROOT ${CMAKE_CURRENT_LIST_DIR}/../..) +include(${LF_ROOT}/core/lf_utils.cmake) + +add_library(lf-network-impl STATIC ${LF_NETWORK_FILES}) +add_library(lf::network-impl ALIAS lf-network-impl) +target_sources(lf-network-impl PUBLIC + ${CMAKE_CURRENT_LIST_DIR}/src/net_util.c + ${CMAKE_CURRENT_LIST_DIR}/src/socket_common.c +) + +if(COMM_TYPE MATCHES TCP) + target_sources(lf-network-impl PUBLIC ${CMAKE_CURRENT_LIST_DIR}/src/lf_socket_support.c) +else() + message(FATAL_ERROR "Your communication type is not supported! The C target supports TCP.") +endif() + +# Link necessary libraries +target_link_libraries(lf-network-impl PUBLIC lf-logging-api) +target_link_libraries(lf-network-impl PUBLIC lf-low-level-platform-api) +target_link_libraries(lf-network-impl PUBLIC lf::tag-api) +target_link_libraries(lf-network-impl PRIVATE lf-network-api) + +lf_enable_compiler_warnings(lf-network-impl) + +target_compile_definitions(lf-network-impl PUBLIC COMM_TYPE_${COMM_TYPE}) diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c new file mode 100644 index 000000000..5e993786c --- /dev/null +++ b/network/impl/src/lf_socket_support.c @@ -0,0 +1,214 @@ +/** + * @file + * @author Edward A. Lee + * @author Dongha Kim + * + * @brief Implementation of socket interface for federated Lingua Franca programs. + */ + +#include // malloc() +#include // strerror() +#include // errno +#include // read() write() +#include // inet_ntop + +#include "net_abstraction.h" +#include "socket_common.h" +#include "util.h" // LF_MUTEX_UNLOCK, "logging.h" + +static socket_priv_t* get_socket_priv_t(net_abstraction_t net_abs) { + if (net_abs == NULL) { + lf_print_error("Network abstraction is already closed."); + return NULL; + } + return (socket_priv_t*)net_abs; +} + +net_abstraction_t initialize_net() { + // Initialize priv. + socket_priv_t* priv = malloc(sizeof(socket_priv_t)); + if (priv == NULL) { + lf_print_error_and_exit("Falied to malloc socket_priv_t."); + } + + // Server initialization. + priv->port = 0; + priv->user_specified_port = 0; + priv->socket_descriptor = -1; + + // Federate initialization + strncpy(priv->server_hostname, "localhost", INET_ADDRSTRLEN); + priv->server_ip_addr.s_addr = 0; + priv->server_port = -1; + + return (net_abstraction_t)priv; +} + +void free_net(net_abstraction_t net_abs) { + socket_priv_t* priv = get_socket_priv_t(net_abs); + free(priv); +} + +int create_server(net_abstraction_t net_abs, bool increment_port_on_retry) { + socket_priv_t* priv = get_socket_priv_t(net_abs); + return create_socket_server(priv->user_specified_port, &priv->socket_descriptor, &priv->port, TCP, + increment_port_on_retry); +} + +net_abstraction_t accept_net(net_abstraction_t server_chan, net_abstraction_t rti_chan) { + socket_priv_t* serv_priv = get_socket_priv_t(server_chan); + int rti_socket; + if (rti_chan == NULL) { + // Set to -1, to indicate that this accept_net() call is not trying to check if the rti_chan is + // available, inside the accept_socket() function. + rti_socket = -1; + } else { + socket_priv_t* rti_priv = get_socket_priv_t(rti_chan); + rti_socket = rti_priv->socket_descriptor; + } + net_abstraction_t fed_net = initialize_net(); + socket_priv_t* fed_priv = get_socket_priv_t(fed_net); + + int sock = accept_socket(serv_priv->socket_descriptor, rti_socket); + if (sock == -1) { + free_net(fed_net); + return NULL; + } + fed_priv->socket_descriptor = sock; + // Get the peer address from the connected socket_id. Saving this for the address query. + if (get_peer_address(fed_priv) != 0) { + lf_print_error("RTI failed to get peer address."); + }; + return fed_net; +} + +void create_client(net_abstraction_t net_abs) { + socket_priv_t* priv = get_socket_priv_t(net_abs); + priv->socket_descriptor = create_real_time_tcp_socket_errexit(); +} + +int connect_to_net(net_abstraction_t net_abs) { + socket_priv_t* priv = get_socket_priv_t(net_abs); + return connect_to_socket(priv->socket_descriptor, priv->server_hostname, priv->server_port); +} + +int read_from_net(net_abstraction_t net_abs, size_t num_bytes, unsigned char* buffer) { + socket_priv_t* priv = get_socket_priv_t(net_abs); + return read_from_socket(priv->socket_descriptor, num_bytes, buffer); +} + +int read_from_net_close_on_error(net_abstraction_t net_abs, size_t num_bytes, unsigned char* buffer) { + socket_priv_t* priv = get_socket_priv_t(net_abs); + int read_failed = read_from_net(net_abs, num_bytes, buffer); + if (read_failed) { + // Read failed. + // Socket has probably been closed from the other side. + // Shut down and close the socket from this side. + shutdown_socket(&priv->socket_descriptor, false); + return -1; + } + return 0; +} + +void read_from_net_fail_on_error(net_abstraction_t net_abs, size_t num_bytes, unsigned char* buffer, char* format, + ...) { + va_list args; + int read_failed = read_from_net_close_on_error(net_abs, num_bytes, buffer); + if (read_failed) { + // Read failed. + if (format != NULL) { + va_start(args, format); + lf_print_error_system_failure(format, args); + va_end(args); + } else { + lf_print_error_system_failure("Failed to read from socket."); + } + } +} + +int write_to_net(net_abstraction_t net_abs, size_t num_bytes, unsigned char* buffer) { + socket_priv_t* priv = get_socket_priv_t(net_abs); + return write_to_socket(priv->socket_descriptor, num_bytes, buffer); +} + +int write_to_net_close_on_error(net_abstraction_t net_abs, size_t num_bytes, unsigned char* buffer) { + socket_priv_t* priv = get_socket_priv_t(net_abs); + int result = write_to_net(net_abs, num_bytes, buffer); + if (result) { + // Write failed. + // Socket has probably been closed from the other side. + // Shut down and close the socket from this side. + shutdown_socket(&priv->socket_descriptor, false); + } + return result; +} + +void write_to_net_fail_on_error(net_abstraction_t net_abs, size_t num_bytes, unsigned char* buffer, lf_mutex_t* mutex, + char* format, ...) { + va_list args; + int result = write_to_net_close_on_error(net_abs, num_bytes, buffer); + if (result) { + // Write failed. + if (mutex != NULL) { + LF_MUTEX_UNLOCK(mutex); + } + if (format != NULL) { + va_start(args, format); + lf_print_error_system_failure(format, args); + va_end(args); + } else { + lf_print_error_and_exit("Failed to write to socket. Shutting down."); + } + } +} + +bool check_net_closed(net_abstraction_t net_abs) { + socket_priv_t* priv = get_socket_priv_t(net_abs); + return check_socket_closed(priv->socket_descriptor); +} + +int shutdown_net(net_abstraction_t net_abs, bool read_before_closing) { + if (net_abs == NULL) { + LF_PRINT_LOG("Socket already closed."); + return 0; + } + socket_priv_t* priv = get_socket_priv_t(net_abs); + int ret = shutdown_socket(&priv->socket_descriptor, read_before_closing); + free_net(net_abs); + return ret; +} + +int32_t get_my_port(net_abstraction_t net_abs) { + socket_priv_t* priv = get_socket_priv_t(net_abs); + return priv->port; +} + +int32_t get_server_port(net_abstraction_t net_abs) { + socket_priv_t* priv = get_socket_priv_t(net_abs); + return priv->server_port; +} + +struct in_addr* get_ip_addr(net_abstraction_t net_abs) { + socket_priv_t* priv = get_socket_priv_t(net_abs); + return &priv->server_ip_addr; +} + +char* get_server_hostname(net_abstraction_t net_abs) { + socket_priv_t* priv = get_socket_priv_t(net_abs); + return priv->server_hostname; +} + +void set_my_port(net_abstraction_t net_abs, int32_t port) { + socket_priv_t* priv = get_socket_priv_t(net_abs); + priv->port = port; +} + +void set_server_port(net_abstraction_t net_abs, int32_t port) { + socket_priv_t* priv = get_socket_priv_t(net_abs); + priv->server_port = port; +} + +void set_server_hostname(net_abstraction_t net_abs, const char* hostname) { + socket_priv_t* priv = get_socket_priv_t(net_abs); + memcpy(priv->server_hostname, hostname, INET_ADDRSTRLEN); +} diff --git a/core/federated/network/net_util.c b/network/impl/src/net_util.c similarity index 97% rename from core/federated/network/net_util.c rename to network/impl/src/net_util.c index ad03c8908..1f0aba0e8 100644 --- a/core/federated/network/net_util.c +++ b/network/impl/src/net_util.c @@ -9,17 +9,14 @@ #include #include #include -#include // For sqrtl() and powl #include // Defines va_list #include #include -#include // Defines memcpy() -#include // Defines nanosleep() -#include // IPPROTO_TCP, IPPROTO_UDP -#include // TCP_NODELAY +#include // Defines memcpy() +#include // Defines nanosleep() #include "net_util.h" -#include "util.h" +#include "logging_macros.h" // Below are more generally useful functions. @@ -58,7 +55,7 @@ void encode_uint16(uint16_t data, unsigned char* buffer) { buffer[1] = (unsigned char)((data & 0xff00) >> 8); } -int host_is_big_endian() { +int host_is_big_endian(void) { static int host = 0; union { uint32_t uint; diff --git a/core/federated/network/socket_common.c b/network/impl/src/socket_common.c similarity index 81% rename from core/federated/network/socket_common.c rename to network/impl/src/socket_common.c index da806a7d2..b8db3f06c 100644 --- a/core/federated/network/socket_common.c +++ b/network/impl/src/socket_common.c @@ -1,6 +1,17 @@ +/** + * @file + * @author Edward A. Lee + * @author Soroush Bateni + * @author Peter Donovan + * @author Dongha Kim + * + * @brief Common socket operations and utilities for federated Lingua Franca programs. + */ + #include // Defines read(), write(), and close() #include // IPPROTO_TCP, IPPROTO_UDP #include // TCP_NODELAY +#include // inet_ntop #include #include #include @@ -10,16 +21,17 @@ #include //va_list #include // strerror -#include "util.h" -#include "socket_common.h" +#include "util.h" // LF_MUTEX_UNLOCK() +#include "logging.h" +#include "net_abstraction.h" /** Number of nanoseconds to sleep before retrying a socket read. */ #define SOCKET_READ_RETRY_INTERVAL 1000000 -// Mutex lock held while performing socket shutdown and close operations. +// Mutex lock held while performing network abstraction shutdown and close operations. lf_mutex_t shutdown_mutex; -int create_real_time_tcp_socket_errexit() { +int create_real_time_tcp_socket_errexit(void) { int sock = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); if (sock < 0) { lf_print_error_system_failure("Could not open TCP socket."); @@ -124,12 +136,12 @@ static int set_socket_bind_option(int socket_descriptor, uint16_t specified_port if (result != 0) { lf_print_error_and_exit("Failed to bind the socket. Port %d is not available. ", used_port); } - lf_print_debug("Socket is binded to port %d.", used_port); + lf_print_debug("Socket is bound to port %d.", used_port); return used_port; } -int create_server(uint16_t port, int* final_socket, uint16_t* final_port, socket_type_t sock_type, - bool increment_port_on_retry) { +int create_socket_server(uint16_t port, int* final_socket, uint16_t* final_port, socket_type_t sock_type, + bool increment_port_on_retry) { int socket_descriptor; struct timeval timeout_time; if (sock_type == TCP) { @@ -165,18 +177,40 @@ int create_server(uint16_t port, int* final_socket, uint16_t* final_port, socket return 0; } -/** - * Return true if either the socket to the RTI is broken or the socket is - * alive and the first unread byte on the socket's queue is MSG_TYPE_FAILED. - */ -static bool check_socket_closed(int socket) { +bool check_socket_closed(int socket) { unsigned char first_byte; ssize_t bytes = peek_from_socket(socket, &first_byte); - if (bytes < 0 || (bytes == 1 && first_byte == MSG_TYPE_FAILED)) { - return true; - } else { - return false; + if (socket > 0) { + if (bytes < 0 || (bytes == 1 && first_byte == MSG_TYPE_FAILED)) { + return true; + } else { + return false; + } + } + lf_print_warning("Socket is no longer connected."); + return true; +} + +int get_peer_address(socket_priv_t* priv) { + struct sockaddr_in peer_addr; + socklen_t addr_len = sizeof(peer_addr); + if (getpeername(priv->socket_descriptor, (struct sockaddr*)&peer_addr, &addr_len) != 0) { + lf_print_error("Failed to get peer address."); + return -1; } + priv->server_ip_addr = peer_addr.sin_addr; + +#if LOG_LEVEL >= LOG_LEVEL_DEBUG + // Create the human readable format and copy that into + // the .server_hostname field of the federate. + char str[INET_ADDRSTRLEN + 1]; + inet_ntop(AF_INET, &priv->server_ip_addr, str, INET_ADDRSTRLEN); + strncpy(priv->server_hostname, str, INET_ADDRSTRLEN - 1); // Copy up to INET_ADDRSTRLEN - 1 characters + priv->server_hostname[INET_ADDRSTRLEN - 1] = '\0'; // Null-terminate explicitly + + LF_PRINT_DEBUG("Got address %s", priv->server_hostname); +#endif + return 0; } int accept_socket(int socket, int rti_socket) { @@ -198,11 +232,12 @@ int accept_socket(int socket, int rti_socket) { break; } else if (errno == EPERM) { lf_print_error_system_failure("Firewall permissions prohibit connection."); + return -1; } else { // For the federates, it should check if the rti_socket is still open, before retrying accept(). if (rti_socket != -1) { if (check_socket_closed(rti_socket)) { - break; + return -1; } } // Try again @@ -296,40 +331,6 @@ int read_from_socket(int socket, size_t num_bytes, unsigned char* buffer) { return 0; } -int read_from_socket_close_on_error(int* socket, size_t num_bytes, unsigned char* buffer) { - assert(socket); - int socket_id = *socket; // Assume atomic read so we don't pass -1 to read_from_socket. - if (socket_id >= 0) { - int read_failed = read_from_socket(socket_id, num_bytes, buffer); - if (read_failed) { - // Read failed. - // Socket has probably been closed from the other side. - // Shut down and close the socket from this side. - shutdown_socket(socket, false); - return -1; - } - return 0; - } - lf_print_warning("Socket is no longer connected. Read failed."); - return -1; -} - -void read_from_socket_fail_on_error(int* socket, size_t num_bytes, unsigned char* buffer, char* format, ...) { - va_list args; - assert(socket); - int read_failed = read_from_socket_close_on_error(socket, num_bytes, buffer); - if (read_failed) { - // Read failed. - if (format != NULL) { - va_start(args, format); - lf_print_error_system_failure(format, args); - va_end(args); - } else { - lf_print_error_system_failure("Failed to read from socket."); - } - } -} - ssize_t peek_from_socket(int socket, unsigned char* result) { ssize_t bytes_read = recv(socket, result, 1, MSG_DONTWAIT | MSG_PEEK); if (bytes_read < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) @@ -364,55 +365,17 @@ int write_to_socket(int socket, size_t num_bytes, unsigned char* buffer) { return 0; } -int write_to_socket_close_on_error(int* socket, size_t num_bytes, unsigned char* buffer) { - assert(socket); - int socket_id = *socket; // Assume atomic read so we don't pass -1 to write_to_socket. - if (socket_id >= 0) { - int result = write_to_socket(socket_id, num_bytes, buffer); - if (result) { - // Write failed. - // Socket has probably been closed from the other side. - // Shut down and close the socket from this side. - shutdown_socket(socket, false); - return -1; - } - return result; - } - lf_print_warning("Socket is no longer connected. Write failed."); - return -1; -} - -void write_to_socket_fail_on_error(int* socket, size_t num_bytes, unsigned char* buffer, lf_mutex_t* mutex, - char* format, ...) { - va_list args; - assert(socket); - int result = write_to_socket_close_on_error(socket, num_bytes, buffer); - if (result) { - // Write failed. - if (mutex != NULL) { - LF_MUTEX_UNLOCK(mutex); - } - if (format != NULL) { - va_start(args, format); - lf_print_error_system_failure(format, args); - va_end(args); - } else { - lf_print_error_and_exit("Failed to write to socket. Shutting down."); - } - } -} - void init_shutdown_mutex(void) { LF_MUTEX_INIT(&shutdown_mutex); } int shutdown_socket(int* socket, bool read_before_closing) { LF_MUTEX_LOCK(&shutdown_mutex); int result = 0; if (*socket < 0) { - lf_print_log("Socket is already closed."); + LF_PRINT_LOG("Socket is already closed."); } else { if (!read_before_closing) { if (shutdown(*socket, SHUT_RDWR)) { - lf_print_log("On shutdown socket, received reply: %s", strerror(errno)); + LF_PRINT_LOG("On shutdown socket, received reply: %s", strerror(errno)); result = -1; } // else shutdown reads and writes succeeded. } else { @@ -420,7 +383,7 @@ int shutdown_socket(int* socket, bool read_before_closing) { // This indicates the write direction is closed. For more details, refer to: // https://stackoverflow.com/questions/4160347/close-vs-shutdown-socket if (shutdown(*socket, SHUT_WR)) { - lf_print_log("Failed to shutdown socket: %s", strerror(errno)); + LF_PRINT_LOG("Failed to shutdown socket: %s", strerror(errno)); result = -1; } else { // Shutdown writes succeeded. @@ -442,7 +405,7 @@ int shutdown_socket(int* socket, bool read_before_closing) { // duplicated packets intended for this program. if (result != 0 && close(*socket)) { // Close failed. - lf_print_log("Error while closing socket: %s\n", strerror(errno)); + LF_PRINT_LOG("Error while closing socket: %s\n", strerror(errno)); result = -1; } *socket = -1;