From 78c05116bb2676857b959bc09b3c49e62b344163 Mon Sep 17 00:00:00 2001 From: harmon25 Date: Sat, 22 Nov 2025 11:06:20 -0500 Subject: [PATCH 1/3] Enhance HTTP request handling with buffering support and improved state management Signed-off-by: harmon25 --- src/httpd.erl | 155 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 102 insertions(+), 53 deletions(-) diff --git a/src/httpd.erl b/src/httpd.erl index 1b1aa0f..d8154d5 100644 --- a/src/httpd.erl +++ b/src/httpd.erl @@ -66,7 +66,8 @@ -record(state, { config, pending_request_map = #{}, - ws_socket_map = #{} + ws_socket_map = #{}, + pending_buffer_map = #{} }). %% @@ -122,59 +123,78 @@ handle_receive(Socket, Packet, State) -> %% @private handle_http_request(Socket, Packet, State) -> - case maps:get(Socket, State#state.pending_request_map, undefined) of + PendingRequestMap = State#state.pending_request_map, + BufferMap = State#state.pending_buffer_map, + PendingBuffer = maps:get(Socket, BufferMap, <<>>), + AccumulatedPacket = <>, + case maps:get(Socket, PendingRequestMap, undefined) of undefined -> - HttpRequest = parse_http_request(binary_to_list(Packet)), - % ?TRACE("HttpRequest: ~p~n", [HttpRequest]), - #{ - method := Method, - headers := Headers - } = HttpRequest, - case get_protocol(Method, Headers) of - http -> - case init_handler(HttpRequest, State) of - {ok, {Handler, HandlerState, PathSuffix, HandlerConfig}} -> - NewHttpRequest = HttpRequest#{ - handler => Handler, - handler_state => HandlerState, - path_suffix => PathSuffix, - handler_config => HandlerConfig, - socket => Socket - }, - handle_request_state(Socket, NewHttpRequest, State); - Error -> - {close, create_error(?INTERNAL_SERVER_ERROR, Error)} - end; - ws -> - ?TRACE("Protocol is ws", []), - Config = State#state.config, - Path = maps:get(path, HttpRequest), - case get_handler(Path, Config) of - {ok, PathSuffix, EntryConfig} -> - WsHandler = maps:get(handler, EntryConfig), - ?TRACE("Got handler ~p", [WsHandler]), - HandlerConfig = maps:get(handler_config, EntryConfig, #{}), - case WsHandler:start(Socket, PathSuffix, HandlerConfig) of - {ok, WebSocket} -> - ?TRACE("Started web socket handler: ~p", [WebSocket]), - NewWebSocketMap = maps:put(Socket, WebSocket, State#state.ws_socket_map), - NewState = State#state{ws_socket_map = NewWebSocketMap}, - ReplyToken = get_reply_token(maps:get(headers, HttpRequest)), - ReplyHeaders = #{"Upgrade" => "websocket", "Connection" => "Upgrade", "Sec-WebSocket-Accept" => ReplyToken}, - Reply = create_reply(?SWITCHING_PROTOCOLS, ReplyHeaders, <<"">>), - ?TRACE("Sending web socket upgrade reply: ~p", [Reply]), - {reply, Reply, NewState}; + case maybe_parse_http_request(AccumulatedPacket) of + {more, IncompletePacket} -> + NewBufferMap = BufferMap#{Socket => IncompletePacket}, + {noreply, State#state{pending_buffer_map = NewBufferMap}}; + {ok, HttpRequest} -> + CleanBufferMap = maps:remove(Socket, BufferMap), + CleanState = State#state{pending_buffer_map = CleanBufferMap}, + % ?TRACE("HttpRequest: ~p~n", [HttpRequest]), + #{ + method := Method, + headers := Headers + } = HttpRequest, + case get_protocol(Method, Headers) of + http -> + case init_handler(HttpRequest, CleanState) of + {ok, {Handler, HandlerState, PathSuffix, HandlerConfig}} -> + NewHttpRequest = HttpRequest#{ + handler => Handler, + handler_state => HandlerState, + path_suffix => PathSuffix, + handler_config => HandlerConfig, + socket => Socket + }, + handle_request_state(Socket, NewHttpRequest, CleanState); Error -> - ?TRACE("Web socket error: ~p", [Error]), - {close, create_error(?INTERNAL_SERVER_ERROR, {web_socket_error, Error})} + {close, create_error(?INTERNAL_SERVER_ERROR, Error)} end; - Error -> - Error - end + ws -> + ?TRACE("Protocol is ws", []), + Config = CleanState#state.config, + Path = maps:get(path, HttpRequest), + case get_handler(Path, Config) of + {ok, PathSuffix, EntryConfig} -> + WsHandler = maps:get(handler, EntryConfig), + ?TRACE("Got handler ~p", [WsHandler]), + HandlerConfig = maps:get(handler_config, EntryConfig, #{}), + case WsHandler:start(Socket, PathSuffix, HandlerConfig) of + {ok, WebSocket} -> + ?TRACE("Started web socket handler: ~p", [WebSocket]), + NewWebSocketMap = maps:put(Socket, WebSocket, CleanState#state.ws_socket_map), + NewState = CleanState#state{ws_socket_map = NewWebSocketMap}, + ReplyToken = get_reply_token(maps:get(headers, HttpRequest)), + ReplyHeaders = #{"Upgrade" => "websocket", "Connection" => "Upgrade", "Sec-WebSocket-Accept" => ReplyToken}, + Reply = create_reply(?SWITCHING_PROTOCOLS, ReplyHeaders, <<"">>), + ?TRACE("Sending web socket upgrade reply: ~p", [Reply]), + {reply, Reply, NewState}; + Error -> + ?TRACE("Web socket error: ~p", [Error]), + {close, create_error(?INTERNAL_SERVER_ERROR, {web_socket_error, Error})} + end; + Error -> + Error + end + end; + {error, Reason} -> + CleanBufferMap = maps:remove(Socket, BufferMap), + _CleanState = State#state{pending_buffer_map = CleanBufferMap}, + {close, create_error(?BAD_REQUEST, Reason)} end; PendingHttpRequest -> ?TRACE("Packetlen: ~p", [erlang:byte_size(Packet)]), - handle_request_state(Socket, PendingHttpRequest#{body := Packet}, State) + ExistingBody = maps:get(body, PendingHttpRequest, <<>>), + NewBody = <>, + CleanBufferMap = maps:remove(Socket, BufferMap), + CleanState = State#state{pending_buffer_map = CleanBufferMap}, + handle_request_state(Socket, PendingHttpRequest#{body := NewBody}, CleanState) end. %% @private @@ -213,7 +233,7 @@ handle_request_state(Socket, HttpRequest, State) -> {reply, Reply, State#state{pending_request_map = NewPendingRequestMap}}; wait_for_body -> NewPendingRequestMap = PendingRequestMap#{Socket => HttpRequest}, - call_http_req_handler(Socket, HttpRequest, State#state{pending_request_map = NewPendingRequestMap}) + {noreply, State#state{pending_request_map = NewPendingRequestMap}} end. %% @private @@ -290,13 +310,19 @@ update_state(Socket, HttpRequest, HandlerState, State) -> %% @hidden handle_tcp_closed(Socket, State) -> - case maps:get(Socket, State#state.ws_socket_map, undefined) of + NewPendingRequestMap = maps:remove(Socket, State#state.pending_request_map), + NewPendingBufferMap = maps:remove(Socket, State#state.pending_buffer_map), + CleanState = State#state{ + pending_request_map = NewPendingRequestMap, + pending_buffer_map = NewPendingBufferMap + }, + case maps:get(Socket, CleanState#state.ws_socket_map, undefined) of undefined -> - State; + CleanState; WebSocket -> ok = httpd_ws_handler:stop(WebSocket), - NewWebSocketMap = maps:remove(Socket, State#state.ws_socket_map), - State#state{ws_socket_map = NewWebSocketMap} + NewWebSocketMap = maps:remove(Socket, CleanState#state.ws_socket_map), + CleanState#state{ws_socket_map = NewWebSocketMap} end. %% @@ -324,6 +350,29 @@ parse_http_request(Packet) -> } ). +maybe_parse_http_request(Packet) when is_binary(Packet) -> + case find_header_delimiter(Packet) of + nomatch -> + {more, Packet}; + {_Pos, _Len} -> + try + {ok, parse_http_request(binary_to_list(Packet))} + catch + throw:Reason -> + {error, Reason}; + error:Reason -> + {error, Reason} + end + end. + +find_header_delimiter(Packet) -> + case binary:match(Packet, <<"\r\n\r\n">>) of + nomatch -> + binary:match(Packet, <<"\n\n">>); + Match -> + Match + end. + %% @private parse_heading([$\s|Rest], start, Tmp, Accum) -> parse_heading(Rest, start, Tmp, Accum); From c7cfe7e75ef860a23cf9c2a4128c462c1cfbb772 Mon Sep 17 00:00:00 2001 From: harmon25 Date: Sun, 30 Nov 2025 14:08:51 -0500 Subject: [PATCH 2/3] fix large bodies (from atomvm_httpd) Signed-off-by: harmon25 --- src/gen_tcp_server.erl | 26 ++++++++++++++++---------- src/httpd.erl | 40 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/src/gen_tcp_server.erl b/src/gen_tcp_server.erl index 64684df..f2c1d30 100644 --- a/src/gen_tcp_server.erl +++ b/src/gen_tcp_server.erl @@ -47,6 +47,7 @@ addr => any }). -define(DEFAULT_SOCKET_OPTIONS, #{}). +-define(MAX_SEND_CHUNK, 2048). %% %% API @@ -168,16 +169,7 @@ try_send(Socket, Packet) when is_binary(Packet) -> "Trying to send binary packet data to socket ~p. Packet (or len): ~p", [ Socket, case byte_size(Packet) < 32 of true -> Packet; _ -> byte_size(Packet) end ]), - case socket:send(Socket, Packet) of - ok -> - ?TRACE("sent.", []), - ok; - {ok, Rest} -> - ?TRACE("sent. remaining: ~p", [Rest]), - try_send(Socket, Rest); - Error -> - io:format("Send failed due to error ~p~n", [Error]) - end; + try_send_binary(Socket, Packet); try_send(Socket, Char) when is_integer(Char) -> %% TODO handle unicode ?TRACE("Sending char ~p as ~p", [Char, <>]), @@ -196,6 +188,20 @@ try_send_iolist(Socket, [H | T]) -> try_send(Socket, H), try_send_iolist(Socket, T). +try_send_binary(_Socket, <<>>) -> + ok; +try_send_binary(Socket, Packet) when is_binary(Packet) -> + ChunkSize = erlang:min(byte_size(Packet), ?MAX_SEND_CHUNK), + <> = Packet, + case socket:send(Socket, Chunk) of + ok -> + try_send_binary(Socket, Rest); + {ok, Remaining} -> + try_send_binary(Socket, <>); + Error -> + io:format("Send failed due to error ~p~n", [Error]) + end. + is_string([]) -> true; is_string([H | T]) when is_integer(H) -> diff --git a/src/httpd.erl b/src/httpd.erl index d8154d5..90137c3 100644 --- a/src/httpd.erl +++ b/src/httpd.erl @@ -570,21 +570,49 @@ create_error(StatusCode, Error) -> create_reply(StatusCode, ContentType, Reply) when is_list(ContentType) orelse is_binary(ContentType) -> create_reply(StatusCode, #{"Content-Type" => ContentType}, Reply); create_reply(StatusCode, Headers, Reply) when is_map(Headers) -> + ReplyLen = iolist_length(Reply), + HeadersWithLen = ensure_content_length(Headers, ReplyLen), [ <<"HTTP/1.1 ">>, erlang:integer_to_binary(StatusCode), <<" ">>, moniker(StatusCode), <<"\r\n">>, - io_lib:format("Server: atomvm-~s\r\n", [get_version_str(erlang:system_info(atomvm_version))]), - to_headers_list(Headers), + io_lib:format("Server: atomvm-~s\r\n", [get_version_str(get_atomvm_version())]), + to_headers_list(HeadersWithLen), <<"\r\n">>, Reply ]. +%% @private +ensure_content_length(Headers, ReplyLen) -> + LenBin = erlang:integer_to_binary(ReplyLen), + CleanHeaders = remove_content_length_header(Headers), + CleanHeaders#{<<"Content-Length">> => LenBin}. + +%% @private +remove_content_length_header(Headers) -> + KeysToRemove = [ + "Content-Length", + <<"Content-Length">>, + "content-length", + <<"content-length">> + ], + lists:foldl(fun(Key, Acc) -> maps:remove(Key, Acc) end, Headers, KeysToRemove). + %% @private maybe_binary_to_string(Bin) when is_binary(Bin) -> erlang:binary_to_list(Bin); maybe_binary_to_string(Other) -> Other. +%% @private +iolist_length(Bin) when is_binary(Bin) -> + erlang:byte_size(Bin); +iolist_length(Int) when is_integer(Int), Int >= 0, Int =< 255 -> + 1; +iolist_length([]) -> + 0; +iolist_length([H | T]) -> + iolist_length(H) + iolist_length(T). + %% @private to_headers_list(Headers) -> [io_lib:format("~s: ~s\r\n", [maybe_binary_to_string(Key), maybe_binary_to_string(Value)]) || {Key, Value} <- maps:to_list(Headers)]. @@ -596,6 +624,14 @@ get_version_str(Version) when is_binary(Version) -> get_version_str(_) -> "unknown". +get_atomvm_version() -> + case catch erlang:system_info(atomvm_version) of + {'EXIT', _} -> + undefined; + Version -> + Version + end. + %% @private moniker(?OK) -> <<"OK">>; From 1bbd2ad1b9295a4c42547a32ce95ba445a290969 Mon Sep 17 00:00:00 2001 From: harmon25 Date: Tue, 16 Dec 2025 09:40:14 -0500 Subject: [PATCH 3/3] implement review suggestions Signed-off-by: harmon25 --- src/httpd.erl | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/src/httpd.erl b/src/httpd.erl index 90137c3..345d64f 100644 --- a/src/httpd.erl +++ b/src/httpd.erl @@ -180,12 +180,10 @@ handle_http_request(Socket, Packet, State) -> {close, create_error(?INTERNAL_SERVER_ERROR, {web_socket_error, Error})} end; Error -> - Error + {close, create_error(?INTERNAL_SERVER_ERROR, {web_socket_error, Error})} end end; {error, Reason} -> - CleanBufferMap = maps:remove(Socket, BufferMap), - _CleanState = State#state{pending_buffer_map = CleanBufferMap}, {close, create_error(?BAD_REQUEST, Reason)} end; PendingHttpRequest -> @@ -575,7 +573,7 @@ create_reply(StatusCode, Headers, Reply) when is_map(Headers) -> [ <<"HTTP/1.1 ">>, erlang:integer_to_binary(StatusCode), <<" ">>, moniker(StatusCode), <<"\r\n">>, - io_lib:format("Server: atomvm-~s\r\n", [get_version_str(get_atomvm_version())]), + io_lib:format("Server: atomvm-~s\r\n", [get_version_str(erlang:system_info(atomvm_version))]), to_headers_list(HeadersWithLen), <<"\r\n">>, Reply @@ -608,10 +606,8 @@ iolist_length(Bin) when is_binary(Bin) -> erlang:byte_size(Bin); iolist_length(Int) when is_integer(Int), Int >= 0, Int =< 255 -> 1; -iolist_length([]) -> - 0; -iolist_length([H | T]) -> - iolist_length(H) + iolist_length(T). +iolist_length(List) when is_list(List) -> + erlang:length(List). %% @private to_headers_list(Headers) -> @@ -624,14 +620,6 @@ get_version_str(Version) when is_binary(Version) -> get_version_str(_) -> "unknown". -get_atomvm_version() -> - case catch erlang:system_info(atomvm_version) of - {'EXIT', _} -> - undefined; - Version -> - Version - end. - %% @private moniker(?OK) -> <<"OK">>;