1

I had a working and running container but after stopping the container and again starting it , it gives me the error:


[ec2-user@ip-10-215-0-163 ~]$ docker start kamailio
Error response from daemon: Cannot start container kamailio: Error getting container 01747db255a74935012442c070945848df706d3880dd6f679e667a3d73461c38 from driver devicemapper: Error mounting '/dev/mapper/docker-202:1-263639-01747db255a74935012442c070945848df706d3880dd6f679e667a3d73461c38' on '/var/lib/docker/devicemapper/mnt/01747db255a74935012442c070945848df706d3880dd6f679e667a3d73461c38': invalid argument
Error: failed to start containers: [kamailio]***

My kernel version, distribution, docker info and docker version:

$ uname -a
Linux ip-10-215-0-163 4.4.5-15.26.amzn1.x86_64 #1 SMP Wed Mar 16 17:15:34 UTC 2016 x86_64 x86_64 x86_64 GNU/Linux

$ docker --version
Docker version 1.9.1, build a34a1d5/1.9.1

$ docker info
Containers: 4
Images: 9
Server Version: 1.9.1
Storage Driver: devicemapper
 Pool Name: docker-202:1-263639-pool
 Pool Blocksize: 65.54 kB
 Base Device Size: 10.74 GB
 Backing Filesystem:
 Data file: /dev/loop0
 Metadata file: /dev/loop1
 Data Space Used: 6.489 GB
 Data Space Total: 107.4 GB
 Data Space Available: 12.68 GB
 Metadata Space Used: 4.444 MB
 Metadata Space Total: 2.147 GB
 Metadata Space Available: 2.143 GB
 Udev Sync Supported: true
 Deferred Removal Enabled: false
 Deferred Deletion Enabled: false
 Deferred Deleted Device Count: 0
 Data loop file: /var/lib/docker/devicemapper/devicemapper/data
 Metadata loop file: /var/lib/docker/devicemapper/devicemapper/metadata
 Library Version: 1.02.93-RHEL7 (2015-01-28)
Execution Driver: native-0.2
Logging Driver: json-file
Kernel Version: 4.4.5-15.26.amzn1.x86_64
Operating System: Amazon Linux AMI 2016.03
CPUs: 1
Total Memory: 3.679 GiB
Name: ip-10-215-0-163
ID: EJNW:5DZK:I7XD:T7LC:WBC2:POG4:3YVS:HK2F:MQFY:INCM:ZEUQ:UC6X

In docker inspect I found this output:

$ docker inspect kamailio
[
{
    "Id": "01747db255a74935012442c070945848df706d3880dd6f679e667a3d73461c38",
    "Created": "2016-04-12T10:36:47.085255232Z",
    "Path": "/usr/bin/supervisord",
    "Args": [],
    "State": {
        "Status": "exited",
        "Running": false,
        "Paused": false,
        "Restarting": false,
        "OOMKilled": false,
        "Dead": false,
        "Pid": 0,
        "ExitCode": 128,
        "Error": "Error getting container 01747db255a74935012442c070945848df706d3880dd6f679e667a3d73461c38 from driver devicemapper: Error mounting '/dev/mapper/docker-202:1-263639-01747db255a74935012442c070945848df706d3880dd6f679e667a3d73461c38' on '/var/lib/docker/devicemapper/mnt/01747db255a74935012442c070945848df706d3880dd6f679e667a3d73461c38': invalid argument",
        "StartedAt": "2016-04-28T12:28:43.482590622Z",
        "FinishedAt": "2016-04-28T18:48:10.249382415Z"
    },
    "Image": "963ddead6a06dbb485ef0bdc8c458b070445b3f52753762f44a8529912f50455",
    "ResolvConfPath": "/var/lib/docker/containers/01747db255a74935012442c070945848df706d3880dd6f679e667a3d73461c38/resolv.conf",
    "HostnamePath": "/var/lib/docker/containers/01747db255a74935012442c070945848df706d3880dd6f679e667a3d73461c38/hostname",
    "HostsPath": "/var/lib/docker/containers/01747db255a74935012442c070945848df706d3880dd6f679e667a3d73461c38/hosts",
    "LogPath": "/var/lib/docker/containers/01747db255a74935012442c070945848df706d3880dd6f679e667a3d73461c38/01747db255a74935012442c070945848df706d3880dd6f679e667a3d73461c38-json.log",
    "Name": "/kamailio",
    "RestartCount": 0,
    "Driver": "devicemapper",
    "ExecDriver": "native-0.2",
    "MountLabel": "",
    "ProcessLabel": "",
    "AppArmorProfile": "",
    "ExecIDs": null,
    "HostConfig": {
        "Binds": null,
        "ContainerIDFile": "",
        "LxcConf": [],
        "Memory": 0,
        "MemoryReservation": 0,
        "MemorySwap": 0,
        "KernelMemory": 0,
        "CpuShares": 0,
        "CpuPeriod": 0,
        "CpusetCpus": "",
        "CpusetMems": "",
        "CpuQuota": 0,
        "BlkioWeight": 0,
        "OomKillDisable": false,
        "MemorySwappiness": -1,
        "Privileged": false,
        "PortBindings": {
            "5060/tcp": [
                {
                    "HostIp": "",
                    "HostPort": "5060"
                }
            ],
            "5060/udp": [
                {
                    "HostIp": "",
                    "HostPort": "5060"
                }
            ],
            "5061/tcp": [
                {
                    "HostIp": "",
                    "HostPort": "5061"
                }
            ],
            "5061/udp": [
                {
                    "HostIp": "",
                    "HostPort": "5061"
                }
            ]
        },
        "Links": null,
        "PublishAllPorts": false,
        "Dns": [],
        "DnsOptions": [],
        "DnsSearch": [],
        "ExtraHosts": null,
        "VolumesFrom": null,
        "Devices": [],
        "NetworkMode": "default",
        "IpcMode": "",
        "PidMode": "",
        "UTSMode": "",
        "CapAdd": null,
        "CapDrop": null,
        "GroupAdd": null,
        "RestartPolicy": {
            "Name": "no",
            "MaximumRetryCount": 0
        },
        "SecurityOpt": null,
        "ReadonlyRootfs": false,
        "Ulimits": null,
        "LogConfig": {
            "Type": "json-file",
            "Config": {}
        },
        "CgroupParent": "",
        "ConsoleSize": [
            0,
            0
        ],
        "VolumeDriver": ""
    },
    "GraphDriver": {
        "Name": "devicemapper",
        "Data": {
            "DeviceId": "271",
            "DeviceName": "docker-202:1-263639-01747db255a74935012442c070945848df706d3880dd6f679e667a3d73461c38",
            "DeviceSize": "10737418240"
        }
    },
    "Mounts": [],
    "Config": {
        "Hostname": "01747db255a7",
        "Domainname": "",
        "User": "",
        "AttachStdin": false,
        "AttachStdout": false,
        "AttachStderr": false,
        "ExposedPorts": {
            "5060/tcp": {},
            "5060/udp": {},
            "5061/tcp": {},
            "5061/udp": {}
        },
        "Tty": false,
        "OpenStdin": false,
        "StdinOnce": false,
        "Env": [
            "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
        ],
        "Cmd": [
            "/usr/bin/supervisord"
        ],
        "Image": "sbc_superd",
        "Volumes": null,
        "WorkingDir": "",
        "Entrypoint": null,
        "OnBuild": null,
        "Labels": {
            "build-date": "2016-03-31",
            "license": "GPLv2",
            "name": "CentOS Base Image",
            "vendor": "CentOS"
        },
        "StopSignal": "SIGTERM"
    },
    "NetworkSettings": {
        "Bridge": "",
        "SandboxID": "",
        "HairpinMode": false,
        "LinkLocalIPv6Address": "",
        "LinkLocalIPv6PrefixLen": 0,
        "Ports": null,
        "SandboxKey": "",
        "SecondaryIPAddresses": null,
        "SecondaryIPv6Addresses": null,
        "EndpointID": "",
        "Gateway": "",
        "GlobalIPv6Address": "",
        "GlobalIPv6PrefixLen": 0,
        "IPAddress": "",
        "IPPrefixLen": 0,
        "IPv6Gateway": "",
        "MacAddress": "",
        "Networks": {
            "bridge": {
                "EndpointID": "",
                "Gateway": "",
                "IPAddress": "",
                "IPPrefixLen": 0,
                "IPv6Gateway": "",
                "GlobalIPv6Address": "",
                "GlobalIPv6PrefixLen": 0,
                "MacAddress": ""
            }
        }
    }
}
]
Tinkaal Gogoi
  • 4,344
  • 4
  • 27
  • 36

2 Answers2

2
service docker stop
thin_check   /var/lib/docker/devicemapper/devicemapper/metadata

Note- if no error then

thin_check  --clear-needs-check-flag   /var/lib/docker/devicemapper/devicemapper/metadata
service docker start

Then we can start the containers

Tinkaal Gogoi
  • 4,344
  • 4
  • 27
  • 36
0

As the storage driver is device-mapper for the container, the container is mounted from /dev/dm-1 in my case.

If we do e2fsck -fv /dev/dm-1 then it will recover the container filesystem and we can start the container.

But still some inodes, files etc. are get deleted so the container might not work properly.

Looking for a better answer

Tinkaal Gogoi
  • 4,344
  • 4
  • 27
  • 36