I am trying to create multiple TUN devices to tunnel the multi-device traffic through a common physical interface.
This is the schema I have:
Socket 1 is sending traffic through tun1 with destiny IP 1.1.1.2 and Socket 2 is doing the same on the other interface.
I have a program running between both TUN devices and the physical interface (eth0) that encapsulates IP packets from the TUN devices into UDP packets and then it sends them to the server. The server unpacks the received packets and answers both clients with echo packets (also encapsulated).
When those packets arrive in eth0, another program reads the packets and forwards them to its TUN device. After that, both programs that are running behind the sockets are blocked on recv() function waiting for the server answer on tunX devices. However, it seems like the kernel is discarding those packets on the TUN devices even when they are well-formed.
I tried by change the netmask of both interfaces to 255.255.255.0 and then only one of the sockets receives correctly the answer.
All this software has been written in C.
This is the function to create a new TUN device:
#define MTU_SIZE 1500
#define SUBNET_MASK 0xFFFFFFFF
int open_tun_iface(char * name, uint8_t * ip)
{
int sock;
struct ifreq ifr_tun;
struct sockaddr_in * sa;
int tunfd = tun_init(name, &ifr_tun, IFF_TUN | IFF_NO_PI);
sock = socket(AF_INET, SOCK_DGRAM, 0);
if (sock < 0) {
perror("Open socket");
tun_close(tunfd);
return -1;
}
/* Set tun IP */
if (set_ip(&ifr_tun, sock, ip) < 0) {
tun_close(tunfd);
return -1;
}
sa = (struct sockaddr_in *)&ifr_tun.ifr_netmask;
memset(sa,0,sizeof(*sa));
sa->sin_family = AF_INET;
sa->sin_addr.s_addr = htonl(SUBNET_MASK);
/* Set the mask */
if (ioctl(sock, SIOCSIFNETMASK, &ifr_tun) < 0)
{
perror("SIOCSIFNETMASK");
tun_close(tunfd);
close(sock);
return -1;
}
/* Read flags */
if (ioctl(sock, SIOCGIFFLAGS, &ifr_tun) < 0) {
perror("SIOCGIFFLAGS");
tun_close(tunfd);
close(sock);
return -1;
}
/* Set Iface up and flags */
ifr_tun.ifr_flags |= IFF_UP;
ifr_tun.ifr_flags |= IFF_RUNNING;
if (ioctl(sock, SIOCSIFFLAGS, &ifr_tun) < 0) {
perror("SIOCGIFFLAGS");
tun_close(tunfd);
close(sock);
return -1;
}
/* Set MTU size */
ifr_tun.ifr_mtu = MTU_SIZE;
if (ioctl(sock, SIOCSIFMTU, &ifr_tun) < 0) {
perror("SIOCSIFMTU");
tun_close(tunfd);
close(sock);
return -1;
}
close(sock);
return tunfd;
}
The function that reads packets from eth0 and forward them to the correct TUN interface is the following one:
void * downlink_distributor(void * args)
{
uint8_t buffer[BUFFER_DATA_LEN];
struct sockaddr_in spgw_addr;
int sockfd;
int sockaddr_len = sizeof(spgw_addr);
int len, packet_len, teid;
eNB * enb = (eNB *) args;
sockfd = get_spgw_socket(enb);
while(1)
{
/* Read packets from the server */
len = recvfrom(sockfd, buffer, BUFFER_DATA_LEN, 0, ( struct sockaddr *) &spgw_addr, (socklen_t *)&sockaddr_len);
if(len < 0)
{
perror("recv downlink_distributor");
return NULL;
}
/* Get the TEID that identifies the TUN device */
teid = analyze_gtp_header(buffer, &packet_len);
if(teid > -1)
{
/* Write the packet in the TUN device associated with the packet TEID */
tun_write(get_tun_device(tun_descriptors[teid % MAX_TUN_DESCRIPTORS]), buffer + 8, packet_len);
}
}
return NULL;
}
Finally, this is the function that is running on both sockets and generate some traffic:
void * _generate_udp_traffic(void * args)
{
uint8_t buffer[BUFFER_DATA_LEN];
struct sockaddr_in dest_addr, spgw_addr;
fd_set fds;
struct timeval timeout;
int retval;
timeout.tv_sec = 0;
timeout.tv_usec = 1;
int tunfd;
char msg[] = "Uplink traffic";
int sockfd;
int len;
int sockaddr_len = sizeof(dest_addr);
udp_generator gen;
/* Copy args */
memcpy(&gen, args, sizeof(udp_generator));
free(args);
/* Get TUN device */
tunfd = get_tun_device(gen.ue);
/* Get UE data socket (TUN socket) */
sockfd = get_data_plane_socket(gen.ue);
/* Configure destiny address */
memset(&dest_addr, 0, sizeof(dest_addr));
dest_addr.sin_family = AF_INET;
dest_addr.sin_port = htons(gen.port_dest);
memcpy((void*) &dest_addr.sin_addr.s_addr, gen.dest_ip, sizeof(struct in_addr));
/* Configure SPGW address */
memset(&spgw_addr, 0, sizeof(spgw_addr));
spgw_addr.sin_family = AF_INET;
spgw_addr.sin_port = htons(S1_U_PORT);
memcpy((void*) &spgw_addr.sin_addr.s_addr, get_spgw_ip(gen.ue), sizeof(struct in_addr));
while(1)
{
sendto(sockfd, msg, strlen(msg), 0, (const struct sockaddr *) &dest_addr, sizeof(dest_addr));
while(1)
{
FD_ZERO(&fds);
FD_SET(tunfd, &fds);
retval = select(tunfd+1, &fds, 0, 0, &timeout);
if (retval == -1){
perror("select()");
}
else if (retval){
/* Reuse of the buffer with enough space for the GTP header */
len = tun_read(tunfd, buffer+8, BUFFER_DATA_LEN); /* We call read to clean the buffer from external traffic */
/* Detect IPv4 packets*/
if((buffer[8] & 0xF0) == 0x40)
{
generate_gtp_header(buffer, gen.ue, len);
break;
}
}
}
/* Protect the access to the eNB socket with a lock*/
pthread_mutex_lock(&lock);
/* Send tho the SPGW */
sendto(get_spgw_socket(gen.enb), buffer, len+8, 0, (const struct sockaddr *) &spgw_addr, sizeof(spgw_addr));
/* Unlock the mutex */
pthread_mutex_unlock(&lock);
/* Receive server answer listening on the TUN device */
len = recvfrom(sockfd, buffer, BUFFER_DATA_LEN, 0, ( struct sockaddr *) &dest_addr, (socklen_t *)&sockaddr_len);
printf("UE %d has received: %s\n", get_ue_id(gen.ue), buffer);
/* Sleep 5 seconds */
sleep(5);
}
return NULL;
}
If the netmask is 32 both programs that are generating client traffic get blocked on recvfrom() inside _generate_udp_traffic function. If I set both netmask to 24 only one of them receive the answer from the server and the other gets stuck at recvfrom.
How can I configure both TUN devices to make both attached sockets work correctly?
UPDATE: I think that my problem is a kernel route table problem. If I use just one socket, I only receive traffic with netmask <= 30. With a netmask of 32, I can see that the server answer is introduced correctly on the TUN interface but somehow, the socket that is blocked in recv() does not receive anything.
