2

On Ubuntu 22.04, I've set up the following iptables rules:

iptables -I OUTPUT -d 192.168.0.0/16 -j LOG --log-prefix "CHECK1 "
iptables -I FORWARD -d 192.168.0.0/16 -j LOG --log-prefix "CHECK2 "

Just to verify that I set things up correctly, I navigated to website.com (which is in the 192.168.0.0/16 subnet) in my browser and saw a CHECK1 message show up in /var/log/kern.log.

I then ran a C program which created a tun interface with an IP address of 172.30.0.1 and emitted a hand-crafted TCP-SYN packet from it. The packet had a source address of 172.30.0.1 and a destination address of 192.168.255.8 (the address of website.com). It showed up in Wireshark. However, there was no corresponding logging message in /var/log/kern.log.

What's happening to this packet?

Here's iptables-save:

*filter
:INPUT ACCEPT [0:0]
:FORWARD DROP [0:0]
:OUTPUT ACCEPT [0:0]
:DOCKER - [0:0]
:DOCKER-ISOLATION-STAGE-1 - [0:0]
:DOCKER-ISOLATION-STAGE-2 - [0:0]
:DOCKER-USER - [0:0]
-A FORWARD -d 192.168.0.0/16 -j LOG --log-prefix "CHECK2 "
-A FORWARD -j DOCKER-USER
-A FORWARD -j DOCKER-ISOLATION-STAGE-1
-A FORWARD -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -o docker0 -j DOCKER
-A FORWARD -i docker0 ! -o docker0 -j ACCEPT
-A FORWARD -i docker0 -o docker0 -j ACCEPT
-A OUTPUT -d 192.168.0.0/16 -j LOG --log-prefix "CHECK1 "
-A DOCKER-ISOLATION-STAGE-1 -i docker0 ! -o docker0 -j DOCKER-ISOLATION-STAGE-2
-A DOCKER-ISOLATION-STAGE-1 -j RETURN
-A DOCKER-ISOLATION-STAGE-2 -o docker0 -j DROP
-A DOCKER-ISOLATION-STAGE-2 -j RETURN
-A DOCKER-USER -j RETURN
COMMIT
*nat
:PREROUTING ACCEPT [0:0]
:INPUT ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
:POSTROUTING ACCEPT [0:0]
:DOCKER - [0:0]
-A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER
-A OUTPUT ! -D 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER
-A POSTROUTING -s 172.17.0.0/16 ! -o docker0 -j MASQUERADE
-A DOCKER -i docker0 -j RETURN
COMMIT

Here's the C code (for brevity's sake, I've omitted error checking):

#include <arpa/inet.h>
#include <fcntl.h>
#include <linux/if.h>
#include <linux/if_tun.h>
#include <netinet/in.h>
#include <stdio.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

static int
tunAlloc(void)
{
    int fd;
    struct ifreq ifr = {.ifr_name = "tun0", .ifr_flags = IFF_TUN | IFF_NO_PI};

    fd = open("/dev/net/tun", O_RDWR);
    ioctl(fd, TUNSETIFF, (void*)&ifr);
    ioctl(fd, TUNSETOWNER, geteuid());
    return fd;
}

static void
bringInterfaceUp(void)
{
    int sock;
    struct sockaddr_in addr = {.sin_family = AF_INET};
    struct ifreq ifr = {.ifr_name = "tun0"};

    inet_aton("172.30.0.1", &addr.sin_addr);
    memcpy(&ifr.ifr_addr, &addr, sizeof(struct sockaddr));

    sock = socket(AF_INET, SOCK_DGRAM, 0);
    ioctl(sock, SIOCSIFADDR, &ifr);
    ioctl(sock, SIOCGIFFLAGS, &ifr);
    ifr.ifr_flags |= IFF_UP | IFF_RUNNING;
    ioctl(sock, SIOCSIFFLAGS, &ifr);
    close(sock);
}

static void
emitPacket(int tap_fd)
{
    unsigned char packet[] = {0x45, 0x00, 0x00, 0x3c, 0xd8, 0x6f, 0x40, 0x00, 0x3f, 0x06, 0x08, 0x91,
                              172,  30,   0,    1,    192,  168,  255,  8,    0xa2, 0x9a, 0x27, 0x11,
                              0x80, 0x0b, 0x63, 0x79, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x02, 0xfa, 0xf0,
                              0x89, 0xd8, 0x00, 0x00, 0x02, 0x04, 0x05, 0xb4, 0x04, 0x02, 0x08, 0x0a,
                              0x5b, 0x76, 0x5f, 0xd4, 0x00, 0x00, 0x00, 0x00, 0x01, 0x03, 0x03, 0x07};

    write(tap_fd, packet, sizeof(packet));
}

int
main()
{
    int tap_fd;

    tap_fd = tunAlloc();
    bringInterfaceUp();
    emitPacket(tap_fd);
    close(tap_fd);

    return 0;
}
8
  • Re: "...and a destination address of 192.168.10.10", what is 192.168.10.10? Is that an address on your host, or elsewhere on the network?
    – larsks
    Feb 14 at 15:57
  • It's elsewhere on the network. Feb 14 at 16:00
  • To be clear, there isn't actually a computer at 192.168.10.10. However, even running nc 192.168.10.10 80 produces logging messages. Feb 14 at 16:09
  • I would expect it to produce only ARP messages, since if the device doesn't exist your local system won't know the MAC address. What happens if you use an address that actually exists? Is the behavior the same?
    – larsks
    Feb 14 at 16:14
  • I get the same behavior. Feb 14 at 16:16

1 Answer 1

5

Using your sample code, I see the same problem you've described.

"Packet, Where Are You?" (pwru) is a great diagnostic tool when packets seem to mysteriously disappear, and when I run:

docker run --privileged --rm -t --pid=host \
  -v /sys/kernel/debug/:/sys/kernel/debug/ \
  cilium/pwru --filter-port 10001

I see the following when your code emits a packet:

0xffff8cd640b80500      5     [sendpacket]        netif_receive_skb
0xffff8cd640b80500      5     [sendpacket]   skb_defer_rx_timestamp
0xffff8cd640b80500      5     [sendpacket]      __netif_receive_skb
0xffff8cd640b80500      5     [sendpacket] __netif_receive_skb_one_core
0xffff8cd640b80500      5     [sendpacket]                   ip_rcv
0xffff8cd640b80500      5     [sendpacket]              ip_rcv_core
0xffff8cd640b80500      5     [sendpacket] kfree_skb_reason(SKB_DROP_REASON_IP_CSUM)
0xffff8cd640b80500      5     [sendpacket]   skb_release_head_state
0xffff8cd640b80500      5     [sendpacket]               sock_wfree
0xffff8cd640b80500      5     [sendpacket]         skb_release_data
0xffff8cd640b80500      5     [sendpacket]            skb_free_head
0xffff8cd640b80500      5     [sendpacket]             kfree_skbmem

That suggests the packet has been dropped because it contains an incorrect checksum.

If we inspect a packet capture with wireshark, it will tell us the correct checksums. Fixing those gets us:

    static void emitPacket(int tap_fd) {
    │ unsigned char packet[] = {
    │   │ 0x45, 0x00, 0x00, 0x3c, 0xd8, 0x6f, 0x40, 0x00, 0x3f, 0x06, 0xf7, 0x7b,
    │   │ 172,  30,   0,    1,    192,  168,  255,  8,    0xa2, 0x9a, 0x27, 0x11,
    │   │ 0x80, 0x0b, 0x63, 0x79, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x02, 0xfa, 0xf0,
    │   │ 0x78, 0xc3, 0x00, 0x00, 0x02, 0x04, 0x05, 0xb4, 0x04, 0x02, 0x08, 0x0a,
    │   │ 0x5b, 0x76, 0x5f, 0xd4, 0x00, 0x00, 0x00, 0x00, 0x01, 0x03, 0x03, 0x07};
    │
    │ write(tap_fd, packet, sizeof(packet));
    }

But this fails in the routing logic:

0xffff8cd8bca97100      0     [sendpacket]        netif_receive_skb
0xffff8cd8bca97100      0     [sendpacket]   skb_defer_rx_timestamp
0xffff8cd8bca97100      0     [sendpacket]      __netif_receive_skb
0xffff8cd8bca97100      0     [sendpacket] __netif_receive_skb_one_core
0xffff8cd8bca97100      0     [sendpacket]                   ip_rcv
0xffff8cd8bca97100      0     [sendpacket]              ip_rcv_core
0xffff8cd8bca97100      0     [sendpacket]               sock_wfree
0xffff8cd8bca97100      0     [sendpacket]             nf_hook_slow
0xffff8cd8bca97100      0     [sendpacket]              nf_checksum
0xffff8cd8bca97100      0     [sendpacket]           nf_ip_checksum
0xffff8cd8bca97100      0     [sendpacket]  __skb_checksum_complete
0xffff8cd8bca97100      0     [sendpacket]       tcp_v4_early_demux
0xffff8cd8bca97100      0     [sendpacket]     ip_route_input_noref
0xffff8cd8bca97100      0     [sendpacket]      ip_route_input_slow
0xffff8cd8bca97100      0     [sendpacket]      fib_validate_source
0xffff8cd8bca97100      0     [sendpacket]    __fib_validate_source
0xffff8cd8bca97100      0     [sendpacket] ip_handle_martian_source
0xffff8cd8bca97101      0     [sendpacket] kfree_skb_reason(SKB_DROP_REASON_NOT_SPECIFIED)
0xffff8cd8bca97100      0     [sendpacket]   skb_release_head_state
0xffff8cd8bca97100      0     [sendpacket]         skb_release_data
0xffff8cd8bca97100      0     [sendpacket]            skb_free_head
0xffff8cd8bca97100      0     [sendpacket]             kfree_skbmem

And in fact if you have log_martians enabled on your system, you'll see:

Feb 14 12:14:03 madhatter kernel: IPv4: martian source 192.168.255.8 from
172.30.0.1, on dev tun0

...and we're getting that error because there's a packet coming in on an interface that has the same address as the interface, which is a problem (we would never be able to respond to it correctly).

If we modify your code to (a) set a different address on tun0 than in the packet, and (b) use a /24 netmask so that we get an automatic route to the appropriate network:

static void bringInterfaceUp(void) {
  int sock;
  struct sockaddr_in addr = {.sin_family = AF_INET};
  struct ifreq ifr = {.ifr_name = "tun0"};

  inet_aton("172.30.0.10", &addr.sin_addr);
  memcpy(&ifr.ifr_addr, &addr, sizeof(struct sockaddr));
  sock = socket(AF_INET, SOCK_DGRAM, 0);
  must(ioctl(sock, SIOCSIFADDR, &ifr));

  /*
  I don't know if this is entirely kosher -- it's the result of a quick
  glance over the netdevice(7) man page -- but it seems to work.
  */
  inet_aton("255.255.255.0", &addr.sin_addr);
  memcpy(&ifr.ifr_addr, &addr, sizeof(struct sockaddr));
  must(ioctl(sock, SIOCSIFNETMASK, &ifr));

  must(ioctl(sock, SIOCGIFFLAGS, &ifr));
  ifr.ifr_flags |= IFF_UP | IFF_RUNNING;
  must(ioctl(sock, SIOCSIFFLAGS, &ifr));
  close(sock);
}

We now see the packet emitted on eth0:

$ sudo tcpdump -i any -nn port 10001
tcpdump: data link type LINUX_SLL2
dropped privs to tcpdump
tcpdump: verbose output suppressed, use -v[v]... for full protocol decode
listening on any, link-type LINUX_SLL2 (Linux cooked v2), snapshot length 262144 bytes
12:33:26.877514 tun0  In  IP 172.30.0.1.41626 > 192.168.255.8.10001: Flags [S], seq 2148230009, win 64240, options [mss 1460,sackOK,TS val 1534484436 ecr 0,nop,wscale 7], length 0
12:33:26.877536 eth0  Out IP 172.30.0.1.41626 > 192.168.255.8.10001: Flags [S], seq 2148230009, win 64240, options [mss 1460,sackOK,TS val 1534484436 ecr 0,nop,wscale 7], length 0
4
  • Is that before it hits any of the netfilter hooks? Also, why would the packet show up in Wireshark? Feb 14 at 17:05
  • 1
    The packet shows up on tun0 in wireshark, because that's where it enters the kernel. I believe this gets dropped before it hits the netfilter hooks.
    – larsks
    Feb 14 at 17:11
  • 2
    Please see my recent updates; I've figured out all the issues and the packet is now successfully emitted on the local network.
    – larsks
    Feb 14 at 17:34
  • Absolutely perfect! Thank you! Feb 14 at 17:43

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.