1

I want to reproduce the pipe system of UNIX Shell in C using execve, dup2, fork, waitpid & pipe functions.

Right, for example this command : /bin/ls -l | /usr/bin/head -2 | /usr/bin/wc is reproduced by :

#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>

int add_child_process(char **argv, int in, int out, char **envp)
{
  int pid;

  pid = fork();
  if (pid == 0)
    {
      if (in != 0)
    {
      dup2(in, 0);
    }
      if (out != 1)
    {
      dup2(out, 1);
    }
      if (execve(argv[0], argv, envp) == -1)
        perror("Execve failed");
    }
  else
    return (pid);


}

int main(int av, char **ag, char **envp)
{
  int in;
  int pipes[2];
  char *argv[3];
  int pids[3];

  /** Launch first process that read on original stdin (0) and write on out of pipe **/
  pipe(pipes);
  argv[0] = "/bin/ls";
  argv[1] = "-l";
  argv[2] = NULL;
  pids[0] = add_child_process(argv, 0, pipes[1], envp);
  close(pipes[1]);
  in = pipes[0];

  /** Launch second process that read on in of old pipe and write on out of new pipe **/
  pipe(pipes);
  argv[0] = "/usr/bin/head";
  argv[1] = "-2";
  argv[2] = NULL;
  pids[1] = add_child_process(argv, in, pipes[1], envp);
  close(in);
  close(pipes[1]);
  in = pipes[0];

  /** Launch last process that read on in of old pipe and write on original stdout (1) **/
  argv[0] = "/usr/bin/wc";
  argv[1] = NULL;
  pids[2] = add_child_process(argv, in, 1, envp);
  close(in);

  /** Wait for all process end to catch all return codes **/
  int return_code;
  waitpid(pids[0], &return_code, 0);
  printf("Process 0 return : %d\n", return_code);
  waitpid(pids[1], &return_code, 0);
  printf("Process 1 return : %d\n", return_code);
  waitpid(pids[2], &return_code, 0);
  printf("Process 2 return : %d\n", return_code);
} 

Work, the output is :

2      11      60
Process 0 return : 0
Process 1 return : 0
Process 2 return : 0

Like echo "/bin/ls -l | /usr/bin/head -2 | /usr/bin/wc" | bash

But for blocking command like ping google.com | head -2 | wc it's show this :

      2      16     145

and remains blocked.

Here is updated main with this command :

int main(int av, char **ag, char **envp)
{
  int in;
  int pipes[2];
  char *argv[3];
  int pids[3];

  /** Launch first process that read on original in (0) and write on out of pipe **/
  pipe(pipes);
  argv[0] = "/bin/ping";
  argv[1] = "google.com";
  argv[2] = NULL;
  pids[0] = add_child_process(argv, 0, pipes[1], envp);
  close(pipes[1]);
  in = pipes[0];

  /** Launch second process that read on in of old pipe and write on out of new pipe **/
  pipe(pipes);
  argv[0] = "/usr/bin/head";
  argv[1] = "-2";
  argv[2] = NULL;
  pids[1] = add_child_process(argv, in, pipes[1], envp);
  close(in);
  close(pipes[1]);
  in = pipes[0];

  /** Launch last process that read on in of old pipe and write on original stdout (1) **/
  argv[0] = "/usr/bin/wc";
  argv[1] = NULL;
  pids[2] = add_child_process(argv, in, 1, envp);
  close(in);

  /** Wait for all process end to catch all return codes **/
  int return_code;
  waitpid(pids[0], &return_code, 0);
  printf("Process 0 return : %d\n", return_code);
  waitpid(pids[1], &return_code, 0);
  printf("Process 1 return : %d\n", return_code);
  waitpid(pids[2], &return_code, 0);
  printf("Process 2 return : %d\n", return_code);
} 

I really don't understand why this behaviour occur for example in Bash : echo "ping google.com | head -2 | wc" | bash show 2 16 145 and don't stuck.

How I can deal with blocking command ? I need to get all return codes of my process for make return according to the last error code.

Rakim Faoui
  • 107
  • 2
  • 8

1 Answers1

1

Your add_child_process() function is supposed to return the PID; it doesn't. It should also have error handling after the execve() in case the program fails to execute.

First solution — not good enough

With those fixed (and sundry other changes to get past my minimum level of compilation warnings — such as #include <stdio.h> so that there's a declaration for printf() before it is used), I get roughly the expected output.

I used this code (source file pc67.c) to test:

#include <assert.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>

static
int add_child_process(char **argv, int in, int out, char **envp)
{
    int pid;

    pid = fork();
    if (pid == 0)
    {
        if (in != 0)
        {
            dup2(in, 0);
        }
        if (out != 1)
        {
            dup2(out, 1);
        }
        execve(argv[0], argv, envp);
        abort();
    }
    return pid;
}

int main(int av, char **ag, char **envp)
{
    int in;
    int pipes[2];
    char *argv[3];
    int pids[3];
    assert(ag[av] == 0);

    /** Launch first process that read on original stdin (0) and write on out of pipe **/
    pipe(pipes);
    argv[0] = "/bin/ls";
    argv[1] = "-l";
    argv[2] = NULL;
    pids[0] = add_child_process(argv, 0, pipes[1], envp);
    close(pipes[1]);
    in = pipes[0];

    /** Launch second process that read on in of old pipe and write on out of new pipe **/
    pipe(pipes);
    argv[0] = "/usr/bin/head";
    argv[1] = "-2";
    argv[2] = NULL;
    pids[1] = add_child_process(argv, in, pipes[1], envp);
    close(in);
    close(pipes[1]);
    in = pipes[0];

    /** Launch last process that read on in of old pipe and write on original stdout (1) **/
    argv[0] = "/usr/bin/wc";
    argv[1] = NULL;
    pids[2] = add_child_process(argv, in, 1, envp);
    close(in);

    /** Wait for all process end to catch all return codes **/
    int return_code;
    waitpid(pids[0], &return_code, 0);
    printf("Process 0 return : %d\n", return_code);
    waitpid(pids[1], &return_code, 0);
    printf("Process 1 return : %d\n", return_code);
    waitpid(pids[2], &return_code, 0);
    printf("Process 2 return : %d\n", return_code);
}

The abort() didn't fire; the commands all executed (I did check that they were in the directories you listed before trying to run the code).

I compile on a Mac running macOS Sierra 10.12.4 using GCC 6.3.0 with the command line:

$ gcc -O3 -g -std=c11 -Wall -Wextra -Werror -Wmissing-prototypes \
>     -Wstrict-prototypes -Wold-style-definition pc67.c -o pc67
$

With that, I get the output:

$ ./pc67
       2      11      70
Process 0 return : 0
Process 1 return : 0
Process 2 return : 0
$

The 11 words are 'total' and the number of blocks, and then 9 words for one line of the ls -l output. I've rerun checking the output from ps before and after the command; there are no stray processes running.

Second solution — circumvention and not cure

Thanks, but unlike Bash echo "ping google.com | head -2 | wc" | bash, it is still blocked in a wait and doesn't terminate the program.

I'm not clear what your alternative command line has to do with your question. However, there are a number of things that could be going on when you introduce ping google.com instead of ls -l as the command. You can't legitimately change the parameters of your question like that. It's a new question altogether.

In the shell surrogate, you've not specified paths to the programs; your code won't handle that. (If you use execvp() instead of execve() — using execve() when you're simply relaying the inherited environment is pointless; the environment is inherited anyway — then the paths to the commands would be irrelevant).

However, using "/sbin/ping" and "google.com" (in a copy of the program pc23.c) does seem to hang. Looking at the setup from another terminal, I see:

$ ps -ftttys000
  UID   PID  PPID   C STIME   TTY           TIME CMD
    0 51551 51550   0 10:13AM ttys000    0:00.50 login -pf jleffler
  502 51553 51551   0 10:13AM ttys000    0:00.14 -bash
  502 54866 51553   0 11:10AM ttys000    0:00.01 pc23
  502 54867 54866   0 11:10AM ttys000    0:00.00 /sbin/ping google.com
  502 54868 54866   0 11:10AM ttys000    0:00.00 (head)
  502 54869 54866   0 11:10AM ttys000    0:00.00 (wc)
$

Both the head and the wc processes have terminated (those are the entries for zombies), but the ping process isn't dying. It doesn't seem to be doing much, but it isn't dying either.

Some time later, I got:

$ ps -ftttys000
  UID   PID  PPID   C STIME   TTY           TIME CMD
    0 51551 51550   0 10:13AM ttys000    0:00.50 login -pf jleffler
  502 51553 51551   0 10:13AM ttys000    0:00.14 -bash
  502 54866 51553   0 11:10AM ttys000    0:00.01 pc23
  502 54867 54866   0 11:10AM ttys000    0:00.09 /sbin/ping google.com
  502 54868 54866   0 11:10AM ttys000    0:00.00 (head)
  502 54869 54866   0 11:10AM ttys000    0:00.00 (wc)
$

It's managed to use 9 CPU seconds. So, in this context, for some reason, ping doesn't pay attention to the SIGPIPE signal.

How to fix? That requires experimentation and could be more intricate. The easiest fix is to add options such as -c and 3 — that worked for me. I don't immediately have an incentive to go looking for alternative fixes.

The macOS man page for ping says, in part:

-c count

Stop after sending (and receiving) count ECHO_RESPONSE packets. If this option is not specified, ping will operate until interrupted. If this option is specified in conjunction with ping sweeps, each sweep will consist of count packets.

It is intriguing to speculate how precise the term 'interrupted' is. The program terminates if sent an actual (Control-C) interrupt, or a SIGTERM signal (kill 54867). It is curious that it sometimes stops on SIGPIPE and sometimes does not.

Third solution — close the file descriptors

On further thought, the problem is that the code is not closing enough file descriptors. That's most often the problem with pipelines not terminating. With ls as the first process, the problem is hidden because ls terminates automatically, closing the stray file descriptors. Changing to ping exposes the problems. Here's another revision of the code. It closes the dup2() descriptors (oops; slap self on wrist). It also ensures that the other pipe descriptor is closed via the new tbc (to be closed) file descriptor argument to add_child_process(). I've also opted to use execv() instead of execve(), removing one argument from the add_child_process() function, and allowing int main(void) since the program pays no attention to its arguments (which allows me to lose the assert(), which was there to 'ensure' that the argument count and vector arguments were 'used', as long as you didn't compile with -DNDEBUG or equivalent).

I also tweaked the command-building code so that it was easy to add/remove the -c and 3 arguments from ping, or to add/remove arguments from other commands.

#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>

static
int add_child_process(char **argv, int in, int out, int tbc)
{
    int pid;

    pid = fork();
    if (pid == 0)
    {
        if (tbc >= 0)
            close(tbc);
        if (in != 0)
        {
            dup2(in, 0);
            close(in);
        }
        if (out != 1)
        {
            dup2(out, 1);
            close(out);
        }
        execv(argv[0], argv);
        abort();
    }
    return pid;
}

int main(void)
{
    int in;
    int pipes[2];
    char *argv[10];
    int pids[3];

    /** Launch first process that read on original stdin (0) and write on out of pipe **/
    pipe(pipes);
    int argn = 0;
    argv[argn++] = "/sbin/ping";
    //argv[argn++] = "-c";
    //argv[argn++] = "3";
    argv[argn++] = "google.com";
    argv[argn++] = NULL;
    pids[0] = add_child_process(argv, 0, pipes[1], pipes[0]);
    close(pipes[1]);
    in = pipes[0];

    /** Launch second process that read on in of old pipe and write on out of new pipe **/
    pipe(pipes);
    argn = 0;
    argv[argn++] = "/usr/bin/head";
    argv[argn++] = "-2";
    argv[argn++] = NULL;
    pids[1] = add_child_process(argv, in, pipes[1], pipes[0]);
    close(in);
    close(pipes[1]);
    in = pipes[0];

    /** Launch last process that read on in of old pipe and write on original stdout (1) **/
    argn = 0;
    argv[argn++] = "/usr/bin/wc";
    argv[argn++] = NULL;
    pids[2] = add_child_process(argv, in, 1, -1);
    close(in);

    /** Wait for all process end to catch all return codes **/
    int return_code;
    waitpid(pids[0], &return_code, 0);
    printf("Process 0 return : %d\n", return_code);
    waitpid(pids[1], &return_code, 0);
    printf("Process 1 return : %d\n", return_code);
    waitpid(pids[2], &return_code, 0);
    printf("Process 2 return : %d\n", return_code);
}

With this code (executable pc73 built from pc73.c), I get output like:

$ ./pc73
       2      14     109
Process 0 return : 13
Process 1 return : 0
Process 2 return : 0
$

There's a short, sub-second pause between the output from wc appearing and the other output; the ping command waits a second before trying to write again. The exit status of 13 indicates ping did die from a SIGPIPE signal. The previous problem was that ping still had the read end of the pipe open, so it didn't get the SIGPIPE signal.

Lesson to be learned — close lots of file descriptors

When working with pipes, make sure you're closing all the file descriptors you need to close.

Community
  • 1
  • 1
Jonathan Leffler
  • 730,956
  • 141
  • 904
  • 1,278
  • Thank's, but unlike Bash `echo "ping google.com | head -2 | wc" | bash` It is still blocked in a wait and don't terminate program [Screenshot](http://image.prntscr.com/image/f41dd38a3dbd46448b2b53b764659f1f.png) – Rakim Faoui Apr 07 '17 at 17:56
  • Yeah when the first process is /bin/ls it's work like Bash, but when it's /bin/ping it didn't work (like my second example). – Rakim Faoui Apr 07 '17 at 18:12
  • See my updated answer. Note that I changed the definition of `argv` to `char *argv[10];` before adding extra arguments to `ping`. I wasn't sure if I'd need to use more arguments in the long run (5 would be enough). – Jonathan Leffler Apr 07 '17 at 18:31
  • Perfect explanation, great ! If I understood correctly when I run my processes one by one connected by a pipe, when the *head* received 2 lines in his stdin he send a *SIGPIPE* to the *ping* to terminate it ? And so when it catch his waitpid function, why status is 0 and not a status like *SIGPIPE* signal ? – Rakim Faoui Apr 07 '17 at 18:43
  • A process gets a `SIGPIPE` signal when it tries to write on a pipe for which there is no reader process. So, after `head` exits, you would expect `ping` to get a `SIGPIPE` when it tries to write to the pipe. It's possible to ignore `SIGPIPE`; then the `write()` calls returns `-1` and sets `errno` to a suitable value (`EPIPE` — 'broken pipe'). But maybe `ping` isn't paying proper attention to the error. – Jonathan Leffler Apr 07 '17 at 18:50
  • Right I understand thanks, I have an other strange behaviour with *echo "man gcc | cat | ls" | tcsh* in TCSH it's don't stuck, but with my program it is [screenshot](http://image.prntscr.com/image/50982559af8e42cbb129e95e0ebf29ea.png) It seem to be a broken pipe (shell return 141), how I can handle this error like TCSH do and avoid infinite wait in my program ? – Rakim Faoui Apr 07 '17 at 19:28
  • I need to amend my answer — not enough file descriptors are being closed. Back in 15 minutes or maybe more. – Jonathan Leffler Apr 07 '17 at 19:34
  • It seems it took me just under 20 minutes to get the information into shape. The code shown in 'Third Solution' takes `ping | head -n 2 | wc` (with paths still required) and it works as you'd expect/want. Check your `tcsh` problem — it likely is also fixed. – Jonathan Leffler Apr 07 '17 at 19:54
  • Very interesting as an answer. In fact you have to close the file descriptor even in the fork? I thought that at the end of the execve the process was killed and the fd closed on its own. And why in the end the -1 for `add_child_process(argv, in, 1, -1, envp);` ? – Rakim Faoui Apr 07 '17 at 20:09
  • That was the trouble for `ping`; it still had the read end of the pipe open, so it wasn't getting the SIGPIPE because there was a process (the same process) that could, in theory, read from the pipe. It was never going to do so, but the o/s doesn't know that the process doesn't know about the file descriptors it has open. So, it assumes it will read from the pipe. And eventually, the `ping` would have filled the pipe and would have then blocked indefinitely waiting for itself to read from the pipe while it was stuck trying to write to it — a livelock scenario. I should've paid more attention. – Jonathan Leffler Apr 07 '17 at 20:12