How to handle interactive apt-get installation with python 3.6 subprocess.Popen() function?

Question

Issue: I wrote the following python 3.6 script using the subprocess.Popen() function to run the linux apt-get command to install debian packages. I encountered a package, ubuntu-restricted-extras, that required user interaction during the installation process. These interactions caused my script to hang.

Question: How can my script avoid hanging during interactive installation and complete the package installation? Interactively where possible. Below is my script. It was tested on Ubuntu 18.04.

import subprocess

def call_subprocess_Popen( cmd, cwd=None ):
    ''' Execute a command in BASH. kwargs: "cmd" is a list.'''
    with subprocess.Popen( cmd, bufsize=1, universal_newlines=True, cwd=cwd,
                           stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                           ) as result:
        for line in result.stdout:
            print( line, end='' )
        for line in result.stderr:
            print( line, end='' )
    print( result.returncode )
    print( result.args )
    if result.returncode != 0:
        raise subprocess.CalledProcessError( result.returncode, result.args )
    else:
        return True

def pkexec_apt_get_y_install( packages ):
    print( f'\npkexec apt_get_y_install ....' )
    cmd = [ 'pkexec', 'apt-get', '-y', 'install' ]
    cmd.extend( packages )
    print( f'cmd = {cmd}' )
    if call_subprocess_Popen( cmd ):
        return True
    else:
        return False

apps = [ 'ubuntu-restricted-extras' ]
pkexec_apt_get_y_install( apps )

Revised script to implement DEBIAN_FRONTEND=noninteractive:

import subprocess, os

def call_subprocess_Popen( cmd, cwd=None, env=None, shell=False ):
    ''' Execute a command in BASH. kwargs: "cmd" is a list.'''
    with subprocess.Popen( cmd, bufsize=1, universal_newlines=True, 
                           stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                           cwd=cwd, env=env, shell=shell ) as result:
        for line in result.stdout:
            print( line, end='' )
        for line in result.stderr:
            print( line, end='' )
    print( result.returncode )
    print( result.args )
    if result.returncode != 0:
        raise subprocess.CalledProcessError( result.returncode, result.args )
    else:
        return True

def pkexec_apt_get_y_install( packages, env=None, shell=False ):
    print( f'\npkexec apt_get_y_install ....' )
    if shell:
        cmd = 'pkexec apt-get -y -q install ' + ' '.join(apps)
    else:
        cmd = [ 'pkexec', 'apt-get', '-y', '-q', 'install' ]
        cmd.extend( packages )
    print( f'cmd = {cmd}' )
    if call_subprocess_Popen( cmd, env=env, shell=shell ):
        return True
    else:
        return False

apps = [ 'ubuntu-restricted-extras' ]
my_env = os.environ.copy()
my_env["DEBIAN_FRONTEND"] = "noninteractive"
pkexec_apt_get_y_install( apps, env=my_env )

Associate Error:

pkexec apt_get_y_install ....
cmd = ['pkexec', 'apt-get', '-y', '-q', 'install', 'ubuntu-restricted-extras']
Reading package lists...
Building dependency tree...
Reading state information...
The following package was automatically installed and is no longer required:
  libllvm7
Use 'apt autoremove' to remove it.
The following additional packages will be installed:
  cabextract gstreamer1.0-fluendo-mp3 gstreamer1.0-libav
  gstreamer1.0-plugins-ugly gstreamer1.0-vaapi i965-va-driver liba52-0.7.4
  libaacs0 libass9 libavcodec-extra libavcodec-extra57 libavfilter6
  libavformat57 libavresample3 libavutil55 libbdplus0 libbluray2 libbs2b0
  libchromaprint1 libcrystalhd3 libdvdnav4 libdvdread4 libflite1 libgme0
  libgsm1 libgstreamer-plugins-bad1.0-0 libmpeg2-4 libmspack0 libmysofa0
  libnorm1 libopenjp2-7 libopenmpt0 libpgm-5.2-0 libpostproc54 librubberband2
  libshine3 libsidplay1v5 libsnappy1v5 libsoxr0 libssh-gcrypt-4 libswresample2
  libswscale4 libva-drm2 libva-wayland2 libva-x11-2 libva2 libvo-amrwbenc0
  libx264-152 libx265-146 libxvidcore4 libzmq5 libzvbi-common libzvbi0
  mesa-va-drivers ttf-mscorefonts-installer ubuntu-restricted-addons unrar
  va-driver-all
Suggested packages:
  gstreamer1.0-vaapi-doc i965-va-driver-shaders libbluray-bdj
  firmware-crystalhd libdvdcss2 sidplay-base
The following NEW packages will be installed:
  cabextract gstreamer1.0-fluendo-mp3 gstreamer1.0-libav
  gstreamer1.0-plugins-ugly gstreamer1.0-vaapi i965-va-driver liba52-0.7.4
  libaacs0 libass9 libavcodec-extra libavcodec-extra57 libavfilter6
  libavformat57 libavresample3 libavutil55 libbdplus0 libbluray2 libbs2b0
  libchromaprint1 libcrystalhd3 libdvdnav4 libdvdread4 libflite1 libgme0
  libgsm1 libgstreamer-plugins-bad1.0-0 libmpeg2-4 libmspack0 libmysofa0
  libnorm1 libopenjp2-7 libopenmpt0 libpgm-5.2-0 libpostproc54 librubberband2
  libshine3 libsidplay1v5 libsnappy1v5 libsoxr0 libssh-gcrypt-4 libswresample2
  libswscale4 libva-drm2 libva-wayland2 libva-x11-2 libva2 libvo-amrwbenc0
  libx264-152 libx265-146 libxvidcore4 libzmq5 libzvbi-common libzvbi0
  mesa-va-drivers ttf-mscorefonts-installer ubuntu-restricted-addons
  ubuntu-restricted-extras unrar va-driver-all
Preconfiguring packages ...
0 upgraded, 59 newly installed, 0 to remove and 0 not upgraded.
Need to get 0 B/28.9 MB of archives.
After this operation, 105 MB of additional disk space will be used.
Selecting previously unselected package libmspack0:amd64.
(Reading database ... 
(Reading database ... 5%
(Reading database ... 10%
(Reading database ... 15%
(Reading database ... 20%
(Reading database ... 25%
(Reading database ... 30%
(Reading database ... 35%
(Reading database ... 40%
(Reading database ... 45%
(Reading database ... 50%
(Reading database ... 55%
(Reading database ... 60%
(Reading database ... 65%
(Reading database ... 70%
(Reading database ... 75%
(Reading database ... 80%
(Reading database ... 85%
(Reading database ... 90%
(Reading database ... 95%
(Reading database ... 100%
(Reading database ... 203579 files and directories currently installed.)
Preparing to unpack .../00-libmspack0_0.6-3ubuntu0.3_amd64.deb ...
Unpacking libmspack0:amd64 (0.6-3ubuntu0.3) ...
Selecting previously unselected package cabextract.
Preparing to unpack .../01-cabextract_1.6-1.1_amd64.deb ...
Unpacking cabextract (1.6-1.1) ...
Selecting previously unselected package ttf-mscorefonts-installer.
Preparing to unpack .../02-ttf-mscorefonts-installer_3.6ubuntu2_all.deb ...
[?1049h[22;0;0t[1;24r[4l[?25l(B[m[37m[40m[1;24r[H[2J[1;1H[97m[45m[K
[K
[K
[K
[K
[K
[K
[K
[K
[K
[K
[K
[K
[K
[K
[K
[K
[K
[K
[K
[K
[K
[K

Arne's Solution:

import os
from subprocess import run, PIPE, STDOUT


def apt_install(pkgs, verbose=True):
    cmd = ['pkexec', 'apt-get', 'install', '-y'] + pkgs
    result = run(
        cmd,
        stdout=PIPE,
        stderr=STDOUT,
        encoding='utf8',
        env={**os.environ, 'DEBIAN_FRONTEND': 'noninteractive'}
    )
    if verbose:
        print("".join(result.stdout))
        print(f"Executed command: {result.args}")
    result.check_returncode()

def setup_msttcorefonts():
    cmd = 'echo msttcorefonts msttcorefonts/{}-mscorefonts-eula {} | debconf-set-selections'
    run(cmd.format("present", "note ''"), shell=True)
    run(cmd.format("accepted", "select true"), shell=True)


# testing with configured licenses, one simple and one complicated package
setup_msttcorefonts()
apt_install(['curl', 'ubuntu-restricted-extras'])

Results from using Arne's Solution:

$ python3.6 Q2_2018_07_14_v1.py 
debconf: DbDriver "passwords" warning: could not open /var/cache/debconf/passwords.dat: Permission denied
debconf: DbDriver "config": could not write /var/cache/debconf/config.dat-new: Permission denied
debconf: DbDriver "passwords" warning: could not open /var/cache/debconf/passwords.dat: Permission denied
debconf: DbDriver "config": could not write /var/cache/debconf/config.dat-new: Permission denied
Error executing command as another user: Request dismissed

Executed command: ['pkexec', 'apt-get', 'install', '-y', 'curl', 'ubuntu-restricted-extras']
Traceback (most recent call last):
  File "Q2_2018_07_14_v1.py", line 27, in <module>
    apt_install(['curl', 'ubuntu-restricted-extras'])
  File "Q2_2018_07_14_v1.py", line 17, in apt_install
    result.check_returncode()
  File "/usr/lib/python3.6/subprocess.py", line 389, in check_returncode
    self.stderr)
subprocess.CalledProcessError: Command '['pkexec', 'apt-get', 'install', '-y', 'curl', 'ubuntu-restricted-extras']' returned non-zero exit status 126.

Answer 1

Running an uninterrupted installation on ubuntu

The installation of ubuntu-restricted-extras requires you to accept its EULA license, which is interactive in order to ensure that a person does the accepting and not a script. So apt-get is behaving correct here by not providing an option to bypass something that is essentially a contract.

But as usual, there is some code that you can run which will skip the interactive step and install ubuntu-restricted-extras anyway.

If you're lucky, setting noninteractive will just work:

DEBIAN_FRONTEND=noninteractive apt-get -y install ubuntu-restricted-extras

Or you can write the info that would be set on see/accept by hand:

echo msttcorefonts msttcorefonts/present-mscorefonts-eula note '' | debconf-set-selections
echo msttcorefonts msttcorefonts/accepted-mscorefonts-eula select true | debconf-set-selections
apt-get -y install ubuntu-restricted-extras

---

As a python script

The following will show how a more or less minimal implementation of an apt installer utility could look like. I'd use subprocess.run instead of subprocess.Popen because it has a nicer interface, as long as you don't you need a very fine-grained control over the process I'd recommend you do the same.

Unless you have accepted the EULA at some other point already, skipping over it with noninteractive will cause your program to print the following lines somewhere in the middle of its run:

mscorefonts-eula license could not be presented 
try 'dpkg-reconfigure debconf' to select a frontend other than noninteractive 
[...]
user did not accept the mscorefonts-eula license

Depending on how exactly your machine is set up, the package might then have been installed but not configured correctly. The ttf-mscorefonts-installer will only run if it was accepted properly before installation, which is why I also had to run an accepting-function before the installation step in all of my tests:

import os
import sys
from subprocess import run


def apt_install(pkgs):
    cmd = ['pkexec', 'apt-get', 'install', '-y'] + pkgs
    print('Running command: {}'.format(' '.join(cmd)))
    result = run(
        cmd,
        stdout=sys.stdout,
        stderr=sys.stderr,
        encoding='utf8',
        env={**os.environ, 'DEBIAN_FRONTEND': 'noninteractive'}
    )
    result.check_returncode()

def accept_eula():
    cmd = 'echo msttcorefonts msttcorefonts/{}-mscorefonts-eula {} | pkexec debconf-set-selections'
    run(cmd.format("present", "note ''"), stdout=sys.stdout, stderr=sys.stderr, shell=True)
    run(cmd.format("accepted", "select true"), stdout=sys.stdout, stderr=sys.stderr, shell=True)


# testing with configured licenses, one simple and one complicated package
accept_eula()
apt_install(['curl', 'ubuntu-restricted-extras'])

Running it like this finally resulted in a properly configured installation that exited with this output:

[...]
All done, no errors.
Extracting cabinet: /var/lib/update-notifier/package-data-downloads/partial/webdin32.exe
  extracting fontinst.exe
  extracting Webdings.TTF
  extracting fontinst.inf
  extracting Licen.TXT

All done, no errors.
All fonts downloaded and installed.
Setting up ttf-mscorefonts-installer (3.6ubuntu2) ...