bugfix: wrong source address for the replying packet when received udp packet via req.socket on secondary address. (#233)

Author: zhuizhuhaomeng

.travis.yml

@@ -11,7 +11,7 @@ compiler:
 
 addons:
   apt:
-    packages: [ axel, cpanminus, libgd-dev, libtest-base-perl, libtext-diff-perl, liburi-perl, libwww-perl, libtest-longstring-perl, liblist-moreutils-perl ]
+    packages: [ axel, cpanminus, libgd-dev, libtest-base-perl, libtext-diff-perl, liburi-perl, libwww-perl, libtest-longstring-perl, liblist-moreutils-perl, dnsutils ]
 
 cache:
   apt: true
@@ -67,6 +67,8 @@ script:
   - sudo iptables -I OUTPUT 1 -p udp --dport 10086 -j REJECT
   - sudo iptables -A OUTPUT -p tcp --dst 127.0.0.2 --dport 12345 -j DROP
   - sudo iptables -A OUTPUT -p udp --dst 127.0.0.2 --dport 12345 -j DROP
+  - sudo ip addr add 10.254.254.1/24 dev lo
+  - sudo ip addr add 10.254.254.2/24 dev lo
   - tar zxf download-cache/pcre-$PCRE_VER.tar.gz
   - cd pcre-$PCRE_VER/
   - ./configure --prefix=$PCRE_PREFIX --enable-jit --enable-utf --enable-unicode-properties > build.log 2>&1 || (cat build.log && exit 1)

src/ngx_stream_lua_socket_udp.c

@@ -71,6 +71,8 @@ static int ngx_stream_lua_socket_udp_close(lua_State *L);
 static ngx_int_t ngx_stream_lua_socket_udp_resume(ngx_stream_lua_request_t *r);
 static void ngx_stream_lua_udp_resolve_cleanup(void *data);
 static void ngx_stream_lua_udp_socket_cleanup(void *data);
+static ssize_t ngx_stream_lua_udp_sendmsg(ngx_connection_t *c,
+    ngx_iovec_t *vec);
 
 
 enum {
@@ -786,6 +788,8 @@ ngx_stream_lua_socket_udp_send(lua_State *L)
     int                                  type;
     const char                          *msg;
     ngx_str_t                            query;
+    ngx_iovec_t                          vec;
+    struct iovec                         iovs[1];
 
     ngx_stream_lua_socket_udp_upstream_t        *u;
     ngx_stream_lua_loc_conf_t                   *llcf;
@@ -921,9 +925,16 @@ ngx_stream_lua_socket_udp_send(lua_State *L)
 
     dd("sending query %.*s", (int) query.len, query.data);
 
-    n = ngx_udp_send(u->udp_connection.connection, query.data, query.len);
+    vec.iovs = iovs;
+    vec.nalloc = 1;
+    vec.count = 1;
+    iovs[0].iov_base = query.data;
+    iovs[0].iov_len = query.len;
+    vec.size = query.len;
+    n = ngx_stream_lua_udp_sendmsg(u->udp_connection.connection, &vec);
 
-    dd("ngx_udp_send returns %d (query len %d)", (int) n, (int) query.len);
+    dd("ngx_stream_lua_udp_sendmsg returns %d (query len %d)",
+       (int) n, (int) query.len);
 
     if (n == NGX_ERROR || n == NGX_AGAIN) {
         u->socket_errno = ngx_socket_errno;
@@ -1797,4 +1808,142 @@ ngx_stream_lua_req_socket_udp(lua_State *L)
 }
 
 
+static ssize_t
+ngx_stream_lua_udp_sendmsg(ngx_connection_t *c, ngx_iovec_t *vec)
+{
+    ssize_t        n;
+    ngx_err_t      err;
+    struct msghdr  msg;
+
+#if (NGX_HAVE_MSGHDR_MSG_CONTROL)
+
+#if (NGX_HAVE_IP_SENDSRCADDR)
+    u_char         msg_control[CMSG_SPACE(sizeof(struct in_addr))];
+#elif (NGX_HAVE_IP_PKTINFO)
+    u_char         msg_control[CMSG_SPACE(sizeof(struct in_pktinfo))];
+#endif
+
+#if (NGX_HAVE_INET6 && NGX_HAVE_IPV6_RECVPKTINFO)
+    u_char         msg_control6[CMSG_SPACE(sizeof(struct in6_pktinfo))];
+#endif
+
+#endif
+
+    ngx_memzero(&msg, sizeof(struct msghdr));
+
+    if (c->socklen) {
+        msg.msg_name = c->sockaddr;
+        msg.msg_namelen = c->socklen;
+    }
+
+    msg.msg_iov = vec->iovs;
+    msg.msg_iovlen = vec->count;
+
+#if (NGX_HAVE_MSGHDR_MSG_CONTROL)
+
+    if (c->listening && c->listening->wildcard && c->local_sockaddr) {
+
+#if (NGX_HAVE_IP_SENDSRCADDR)
+
+        if (c->local_sockaddr->sa_family == AF_INET) {
+            struct cmsghdr      *cmsg;
+            struct in_addr      *addr;
+            struct sockaddr_in  *sin;
+
+            msg.msg_control = &msg_control;
+            msg.msg_controllen = sizeof(msg_control);
+
+            cmsg = CMSG_FIRSTHDR(&msg);
+            cmsg->cmsg_level = IPPROTO_IP;
+            cmsg->cmsg_type = IP_SENDSRCADDR;
+            cmsg->cmsg_len = CMSG_LEN(sizeof(struct in_addr));
+
+            sin = (struct sockaddr_in *) c->local_sockaddr;
+
+            addr = (struct in_addr *) CMSG_DATA(cmsg);
+            *addr = sin->sin_addr;
+        }
+
+#elif (NGX_HAVE_IP_PKTINFO)
+
+        if (c->local_sockaddr->sa_family == AF_INET) {
+            struct cmsghdr      *cmsg;
+            struct in_pktinfo   *pkt;
+            struct sockaddr_in  *sin;
+
+            msg.msg_control = &msg_control;
+            msg.msg_controllen = sizeof(msg_control);
+
+            cmsg = CMSG_FIRSTHDR(&msg);
+            cmsg->cmsg_level = IPPROTO_IP;
+            cmsg->cmsg_type = IP_PKTINFO;
+            cmsg->cmsg_len = CMSG_LEN(sizeof(struct in_pktinfo));
+
+            sin = (struct sockaddr_in *) c->local_sockaddr;
+
+            pkt = (struct in_pktinfo *) CMSG_DATA(cmsg);
+            ngx_memzero(pkt, sizeof(struct in_pktinfo));
+            pkt->ipi_spec_dst = sin->sin_addr;
+        }
+
+#endif
+
+#if (NGX_HAVE_INET6 && NGX_HAVE_IPV6_RECVPKTINFO)
+
+        if (c->local_sockaddr->sa_family == AF_INET6) {
+            struct cmsghdr       *cmsg;
+            struct in6_pktinfo   *pkt6;
+            struct sockaddr_in6  *sin6;
+
+            msg.msg_control = &msg_control6;
+            msg.msg_controllen = sizeof(msg_control6);
+
+            cmsg = CMSG_FIRSTHDR(&msg);
+            cmsg->cmsg_level = IPPROTO_IPV6;
+            cmsg->cmsg_type = IPV6_PKTINFO;
+            cmsg->cmsg_len = CMSG_LEN(sizeof(struct in6_pktinfo));
+
+            sin6 = (struct sockaddr_in6 *) c->local_sockaddr;
+
+            pkt6 = (struct in6_pktinfo *) CMSG_DATA(cmsg);
+            ngx_memzero(pkt6, sizeof(struct in6_pktinfo));
+            pkt6->ipi6_addr = sin6->sin6_addr;
+        }
+
+#endif
+    }
+
+#endif
+
+eintr:
+
+    n = sendmsg(c->fd, &msg, 0);
+
+    ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "sendto: fd:%d %z of %uz to \"%V\"",
+                   c->fd, n, vec->size, &c->addr_text);
+    if (n == -1) {
+        err = ngx_errno;
+
+        switch (err) {
+        case NGX_EAGAIN:
+            ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,
+                           "sendmsg() not ready");
+            return NGX_AGAIN;
+
+        case NGX_EINTR:
+            ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,
+                           "sendmsg() was interrupted");
+            goto eintr;
+
+        default:
+            c->write->error = 1;
+            ngx_connection_error(c, err, "sendmsg() failed");
+            return NGX_ERROR;
+        }
+    }
+
+    return n;
+}
+
 /* vi:set ft=c ts=4 sw=4 et fdm=marker: */

t/142-req-udp-socket.t

@@ -243,3 +243,43 @@ hello world! my
 received: hello world! my
 --- no_error_log
 [error]
+
+
+
+=== TEST 7: request on secondary ip address
+sudo ip addr add 10.254.254.1/24 dev lo
+sudo ip addr add 10.254.254.2/24 dev lo
+nmap will be blocked by travis , use dig to send dns request.
+--- dgram_server_config
+    content_by_lua_block {
+        local sock, err = ngx.req.socket()
+        if not sock then
+            ngx.log(ngx.ERR,"ngx.req.socket error : ", err)
+            return ngx.exit(ngx.ERROR)
+        end
+
+        local data = sock:receive()
+        local ok, err = sock:send(data)
+        if not ok then
+            ngx.log(ngx.ERR, "failed to send: ", err)
+            return ngx.exit(ngx.ERROR)
+        end
+    }
+--- config
+     location = /dns {
+         content_by_lua_block {
+            local cmd = "dig -b 10.254.254.1 @10.254.254.2 www.baidu.com -p " .. tostring(ngx.var.server_port + 1)
+            local f = io.popen(cmd, "r")
+            ngx.sleep(0.2)
+            local result = f:read("*a")
+            f:close()
+            ngx.say("hello")
+         }
+     }
+--- request
+GET /dns
+--- response_body
+hello
+--- grep_error_log eval: qr/sendto: fd.*$/
+--- grep_error_log_out eval
+qr/sendto: fd:\d+ \d+ of \d+ to "10.254.254.1"/

valgrind.suppress

@@ -196,3 +196,25 @@
    fun:ngx_set_environment
    fun:ngx_single_process_cycle
 }
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Param
+   sendmsg(msg.msg_control)
+   fun:sendmsg
+   fun:ngx_stream_lua_udp_sendmsg
+   fun:ngx_stream_lua_socket_udp_send
+   fun:lj_BC_FUNCC
+   fun:ngx_stream_lua_run_thread
+   fun:ngx_stream_lua_content_by_chunk
+   fun:ngx_stream_lua_content_handler_inline
+   fun:ngx_stream_lua_content_handler
+   fun:ngx_stream_core_content_phase
+   fun:ngx_stream_core_run_phases
+   fun:ngx_stream_session_handler
+   fun:ngx_stream_init_connection
+   fun:ngx_event_recvmsg
+   fun:ngx_epoll_process_events
+   fun:ngx_process_events_and_timers
+   fun:ngx_single_process_cycle
+   fun:main
+}