Continuously increasing map ids in eBPF

Question

I am running eBPF sockops program. During testing, I need to load and reload the programs several times.

Here is the userspace program I use to load and attach the BPF program:

static const char *__doc__ = "User space program to load the sockops bpf program to register sockets\n";

#include <bpf/bpf.h>
#include <bpf/libbpf.h>
#include <errno.h>
#include <fcntl.h>
#include <string.h>
#include <unistd.h>

const char *cgroup_dir = "/sys/fs/cgroup/unified";

int main(int argc, char **argv) {
  int err, len;
  int cgroup_fd;
  const char *sockops_file = "bpf_sockops.o";
  struct bpf_object *sockhash_obj;
  bool do_unload = false;

  // Open the cgroup fd -- needed for both attach and detach operations.
  fprintf(stdout, "Opening cgroup file %s\n", cgroup_filename);
  cgroup_fd = open(cgroup_dir, O_RDONLY);
  if (cgroup_fd < 0) {
    fprintf(stderr, "ERR: opening cgroup file %s\n", strerror(errno));
    goto exit_cgroup;
  }

  // Check if the program is to be unloaded.
  if (do_unload) {
    // Unload the sockops program
    err = bpf_prog_detach(cgroup_fd, BPF_CGROUP_SOCK_OPS);
    if (err) {
      goto fail;
    }

    return 0;
  }

  // Open, load and attach sockops_obj if not already attached
  sockhash_obj = bpf_object__open_file(sockops_file, NULL);
  if (libbpf_get_error(sockhash_obj)) {
    goto fail;
  }

  struct bpf_program *sockops_prog = bpf_object__find_program_by_name(sockhash_obj, "bpf_add_to_sockhash");
  if (!sockops_prog) {
    goto fail;
  }

  // Load the sockops program
  err = bpf_object__load(sockhash_obj);
  if (err) {
    goto fail;
  }

  // Attach the sockops program
  // Using core BPF API as libbpf doesn't support sockops yet.
  err = bpf_prog_attach(bpf_program__fd(sockops_prog), cgroup_fd, BPF_CGROUP_SOCK_OPS, 0);
  if (err) {
    fprintf(stderr, "ERR: attaching program\n");
    goto fail;
  }

  fprintf(stdout, "Successfully loaded BPF program.\n");

  return 0;

exit_cgroup:
  close(cgroup_fd);

fail:
  return -1;
}

The key calls that I make to attach/detach the sockops program from the userspace program are:

err = bpf_prog_attach(bpf_program__fd(sockops_prog), cgroup_fd, BPF_CGROUP_SOCK_OPS, 0);
...
...
...
err = bpf_prog_detach(cgroup_fd, BPF_CGROUP_SOCK_OPS);

My sockops program looks like:

// File Name: bpf_sockops.c
#include <linux/in.h>
#include <linux/tcp.h>

#include <linux/bpf.h>
#include <sys/socket.h>

#include <bpf/bpf_endian.h>
#include <bpf/bpf_helpers.h>

char LICENSE[] SEC("license") = "GPL";

// sock_ops_map maps the sock_ops key to a socket descriptor
struct {
  __uint(type, BPF_MAP_TYPE_SOCKHASH);
  __uint(max_entries, 65535);
  __type(key, struct sock_key);
  __type(value, __u64);
} sock_ops_map SEC(".maps");

// `sock_key' is a key for the sockmap
struct sock_key {
  __u32 sip4;
  __u32 dip4;
  __u32 sport;
  __u32 dport;
} __attribute__((packed));

// `sk_extract_key' extracts the key from the `bpf_sock_ops' struct
static inline void sk_extract_key(struct bpf_sock_ops *ops,
                                  struct sock_key *key) {
  key->dip4 = ops->remote_ip4;
  key->sip4 = ops->local_ip4;
  key->sport = (bpf_htonl(ops->local_port) >> 16);
  key->dport = ops->remote_port >> 16;
}

SEC("sockops")
int bpf_add_to_sockhash(struct bpf_sock_ops *skops) {
  __u32 family, op;

  family = skops->family;
  op = skops->op;
  bpf_printk("Got new operation %d for socket.\n", op);

  switch (op) {
    case BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB:
    case BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB:
      if (family == AF_INET) {
        struct sock_key key = {};
        sk_extract_key(skops, &key);
        int ret = bpf_sock_hash_update(skops, &sock_ops_map, &key, BPF_NOEXIST);
        if (ret != 0) {
          bpf_printk("Failed to update sockmap: %d\n", ret);
        } else {
          bpf_printk("Added new socket to sockmap\n");
        }
      }
      break;
    default:
      break;
  }
  return 0;
}

As I reload the program several times, the map id for the sock_ops_map keeps on increasing:

~/ebpf-code$ sudo bpftool prog show
1751: cgroup_skb  tag 6deef7357e7b4530  gpl
        loaded_at 2023-06-21T23:42:28-0600  uid 0
        xlated 64B  jited 54B  memlock 4096B
1752: cgroup_skb  tag 6deef7357e7b4530  gpl
        loaded_at 2023-06-21T23:42:28-0600  uid 0
        xlated 64B  jited 54B  memlock 4096B
1753: cgroup_skb  tag 6deef7357e7b4530  gpl
        loaded_at 2023-06-21T23:42:28-0600  uid 0
        xlated 64B  jited 54B  memlock 4096B
1754: cgroup_skb  tag 6deef7357e7b4530  gpl
        loaded_at 2023-06-21T23:42:28-0600  uid 0
        xlated 64B  jited 54B  memlock 4096B
1755: cgroup_skb  tag 6deef7357e7b4530  gpl
        loaded_at 2023-06-21T23:42:28-0600  uid 0
        xlated 64B  jited 54B  memlock 4096B
1756: cgroup_skb  tag 6deef7357e7b4530  gpl
        loaded_at 2023-06-21T23:42:28-0600  uid 0
        xlated 64B  jited 54B  memlock 4096B
1762: sock_ops  name bpf_add_to_sock  tag 79fcc2296545280f  gpl
        loaded_at 2023-06-21T23:53:36-0600  uid 0
        xlated 880B  jited 521B  memlock 4096B  map_ids 1219,1217,1220
        btf_id 1554

As shown above, the map ids are already 1217-1220. Is it expected for the map ids to be continuously increasing? What happens when the limit (or max map id) is reached?

Dylan Reimerink · Accepted Answer · 2023-06-23T07:25:50.560

2

Is it expected for the map ids to be continuously increasing?

Maps are unloaded as soon as the last reference to the map goes away. So when you detach the old program with bpf_prog_detach you also cause the map to unload. Then when you bpf_object__load you re-create the map thus incrementing its ID.

So it depends on your goals if this is expected or not. If you don't want this to happen you can do the following:

The easiest way to persist the map across program reloads is to pin the map. You can do so my setting the pinning field on the map:

struct {
  __uint(type, BPF_MAP_TYPE_SOCKHASH);
  __uint(max_entries, 65535);
  __type(key, struct sock_key);
  __type(value, __u64);
  __uint(pinning, LIBBPF_PIN_BY_NAME);
} sock_ops_map SEC(".maps");

You will need to specify a pinning path when opening the object file with bpf_object__open_file. You have to set opts->pin_root_path.

What happens when the limit (or max map id) is reached?

The ids will increase until (1 << 32)-1 then rollover and start from scratch, the kernel will re-use ids that are no longer in use, but never re-use an id if it still exists

edited Jun 23 '23 at 07:25

answered Jun 23 '23 at 07:02

Dylan Reimerink

5,874
2
15
21

Actually, I am pinning the map in a file in the bpf FS. However, since the sockets will be updated for new runs, I explicitly `rm -r` the bpf directory where I pin the maps. Do you suggest I should let the previous entries in the SOCKHASH remain? – diviquery Jun 23 '23 at 07:18
If you do not want to persist the data in the map between runs then re-creating the map is not a bad thing. I only suggest re-using the pinned maps if you want to persist data between program reloads. – Dylan Reimerink Jun 23 '23 at 07:23
Okay, makes sense. Can there be an issue because of the ever-increasing map IDs? – diviquery Jun 23 '23 at 07:26
No, not a problem, updated the answer with more details. – Dylan Reimerink Jun 23 '23 at 07:27

Continuously increasing map ids in eBPF

1 Answers1