So… you do basic dist upgrade.. you don’t really make an effort to take a cloud-server image (because your me, and are an idiot! and dont even follow the advice you give your own customers, yourself!)…
Now I’ve got that bit over.. I basically found that udev wasn’t installed. This might not seem like completely the end of the world, however, the fact that /dev/random /dev/urandom were missing, and /dev/null was a regular file.. we had big issues man.. I didn’t think I’d be able to get SSH to behave again, but actually it seems this is isolated to udev in most latest centos revisions.
I’m not going to give my kernel version, but you get the idea ;-D
[root@RESCUE-pirax-test ~]# ls -al total 40 dr-xr-x---. 5 root root 4096 Dec 7 09:34 . dr-xr-xr-x. 18 root root 4096 Dec 7 09:34 .. -rw-r--r--. 1 root root 18 Dec 29 2013 .bash_logout -rw-r--r--. 1 root root 176 Dec 29 2013 .bash_profile -rw-r--r--. 1 root root 176 Dec 29 2013 .bashrc drwxr-xr-x. 3 root root 4096 Dec 7 09:34 .cache drwxr-xr-x. 3 root root 4096 Dec 7 09:34 .config -rw-r--r--. 1 root root 100 Dec 29 2013 .cshrc drwx------. 2 root root 4096 Dec 7 09:34 .ssh -rw-r--r--. 1 root root 129 Dec 29 2013 .tcshrc [root@RESCUE-pirax-test ~]# cd /mnt [root@RESCUE-pirax-test mnt]# ls -al total 112 dr-xr-xr-x. 23 root root 4096 Dec 7 09:32 . dr-xr-xr-x. 18 root root 4096 Dec 7 09:34 .. lrwxrwxrwx. 1 root root 7 Feb 23 2016 bin -> usr/bin dr-xr-xr-x. 4 root root 4096 Dec 6 11:53 boot drwxr-xr-x. 2 root root 4096 Apr 21 2016 customer drwxr-xr-x. 2 root root 4096 Dec 7 09:47 dev drwxr-xr-x. 8 root root 4096 Jun 10 10:02 documents drwxr-xr-x. 3 root root 4096 Mar 1 2016 dump drwxr-xr-x. 91 root root 12288 Dec 7 09:45 etc drwxr-xr-x. 5 root root 4096 Nov 11 11:58 home drwxr-xr-x. 3 root root 4096 Nov 9 2015 include lrwxrwxrwx. 1 root root 7 Feb 23 2016 lib -> usr/lib lrwxrwxrwx. 1 root root 9 Feb 23 2016 lib64 -> usr/lib64 drwx------. 2 root root 16384 Sep 3 2015 lost+found drwxr-xr-x. 2 root root 4096 Aug 12 2015 media drwxr-xr-x. 3 root root 4096 Jun 23 14:23 mnt drwxr-xr-x. 4 root root 4096 Aug 12 2015 opt drwxr-xr-x. 2 root root 4096 Sep 3 2015 proc drwxr-xr-x. 26 root root 4096 Dec 7 09:36 root drwxr-xr-x. 3 root root 4096 Dec 7 09:43 run lrwxrwxrwx. 1 root root 8 Feb 23 2016 sbin -> usr/sbin drwxr-xr-x. 2 root root 4096 Aug 12 2015 srv drwxr-xr-x. 2 root root 4096 Sep 3 2015 sys drwxrwxrwt. 7 root root 4096 Dec 7 09:46 tmp drwxr-xr-x. 13 root root 4096 Feb 23 2016 usr drwxr-xr-x. 22 root root 4096 Dec 7 09:31 var drwxr-xr-x. 9 root root 4096 Oct 12 2015 wpscan [root@RESCUE-pirax-test mnt]# cd .. [root@RESCUE-pirax-test /]# chroot /mnt [root@RESCUE-pirax-test /]# ls -al total 112 dr-xr-xr-x. 23 root root 4096 Dec 7 09:32 . dr-xr-xr-x. 23 root root 4096 Dec 7 09:32 .. lrwxrwxrwx. 1 root root 7 Feb 23 2016 bin -> usr/bin dr-xr-xr-x. 4 root root 4096 Dec 6 11:53 boot drwxr-xr-x. 2 root root 4096 Apr 21 2016 customer drwxr-xr-x. 2 root root 4096 Dec 7 09:47 dev drwxr-xr-x. 8 root root 4096 Jun 10 10:02 documents drwxr-xr-x. 3 root root 4096 Mar 1 2016 dump drwxr-xr-x. 91 root root 12288 Dec 7 09:45 etc drwxr-xr-x. 5 root root 4096 Nov 11 11:58 home drwxr-xr-x. 3 root root 4096 Nov 9 2015 include lrwxrwxrwx. 1 root root 7 Feb 23 2016 lib -> usr/lib lrwxrwxrwx. 1 root root 9 Feb 23 2016 lib64 -> usr/lib64 drwx------. 2 root root 16384 Sep 3 2015 lost+found drwxr-xr-x. 2 root root 4096 Aug 12 2015 media drwxr-xr-x. 3 root root 4096 Jun 23 14:23 mnt drwxr-xr-x. 4 root root 4096 Aug 12 2015 opt drwxr-xr-x. 2 root root 4096 Sep 3 2015 proc drwxr-xr-x. 26 root root 4096 Dec 7 09:36 root drwxr-xr-x. 3 root root 4096 Dec 7 09:43 run lrwxrwxrwx. 1 root root 8 Feb 23 2016 sbin -> usr/sbin drwxr-xr-x. 2 root root 4096 Aug 12 2015 srv drwxr-xr-x. 2 root root 4096 Sep 3 2015 sys drwxrwxrwt. 7 root root 4096 Dec 7 09:46 tmp drwxr-xr-x. 13 root root 4096 Feb 23 2016 usr drwxr-xr-x. 22 root root 4096 Dec 7 09:31 var drwxr-xr-x. 9 root root 4096 Oct 12 2015 wpscan [root@RESCUE-pirax-test /]# su adam [adam@RESCUE-pirax-test /]$ ssh root@localhost cannot read from /dev/urandom, No such file or directory [adam@RESCUE-pirax-test /]$ yum update mkinitrd Loaded plugins: fastestmirror, langpacks You need to be root to perform this command. [adam@RESCUE-pirax-test /]$ exit exit [root@RESCUE-pirax-test /]# yum update mkinitrd Loaded plugins: fastestmirror, langpacks Loading mirror speeds from cached hostfile * base: mirrors.vooservers.com * epel: epel.check-update.co.uk * extras: mirrors.vooservers.com * updates: mirrors.vooservers.com No Match for argument: mkinitrd No package mkinitrd available. No packages marked for update [root@RESCUE-pirax-test /]# yum provides initrd Loaded plugins: fastestmirror, langpacks Loading mirror speeds from cached hostfile * base: mirrors.vooservers.com * epel: epel.check-update.co.uk * extras: mirrors.vooservers.com * updates: mirrors.vooservers.com No matches found [root@RESCUE-pirax-test /]# cd /dev [root@RESCUE-pirax-test dev]# /sbin/MAKEDEV urandom; bash: /sbin/MAKEDEV: No such file or directory [root@RESCUE-pirax-test dev]# yum install udev Loaded plugins: fastestmirror, langpacks Loading mirror speeds from cached hostfile * base: mirrors.vooservers.com * epel: epel.check-update.co.uk * extras: mirrors.vooservers.com * updates: mirrors.vooservers.com Resolving Dependencies --> Running transaction check ---> Package systemd.x86_64 0:219-19.el7_2.4 will be updated --> Processing Dependency: systemd = 219-19.el7_2.4 for package: systemd-python-219-19.el7_2.4.x86_64 --> Processing Dependency: systemd = 219-19.el7_2.4 for package: systemd-sysv-219-19.el7_2.4.x86_64 ---> Package systemd.x86_64 0:219-19.el7_2.13 will be an update --> Processing Dependency: systemd-libs = 219-19.el7_2.13 for package: systemd-219-19.el7_2.13.x86_64 --> Running transaction check ---> Package systemd-libs.x86_64 0:219-19.el7_2.4 will be updated --> Processing Dependency: systemd-libs = 219-19.el7_2.4 for package: libgudev1-219-19.el7_2.4.x86_64 ---> Package systemd-libs.x86_64 0:219-19.el7_2.13 will be an update ---> Package systemd-python.x86_64 0:219-19.el7_2.4 will be updated ---> Package systemd-python.x86_64 0:219-19.el7_2.13 will be an update ---> Package systemd-sysv.x86_64 0:219-19.el7_2.4 will be updated ---> Package systemd-sysv.x86_64 0:219-19.el7_2.13 will be an update --> Running transaction check ---> Package libgudev1.x86_64 0:219-19.el7_2.4 will be updated ---> Package libgudev1.x86_64 0:219-19.el7_2.13 will be an update --> Finished Dependency Resolution Dependencies Resolved ========================================================================================================================================================================================================================================================================= Package Arch Version Repository Size ========================================================================================================================================================================================================================================================================= Updating: systemd x86_64 219-19.el7_2.13 updates 5.1 M Updating for dependencies: libgudev1 x86_64 219-19.el7_2.13 updates 67 k systemd-libs x86_64 219-19.el7_2.13 updates 358 k systemd-python x86_64 219-19.el7_2.13 updates 100 k systemd-sysv x86_64 219-19.el7_2.13 updates 54 k Transaction Summary ========================================================================================================================================================================================================================================================================= Upgrade 1 Package (+4 Dependent packages) Total size: 5.7 M Is this ok [y/d/N]: y Downloading packages: Running transaction check Running transaction test Transaction test succeeded Running transaction Updating : systemd-libs-219-19.el7_2.13.x86_64 1/10 Updating : systemd-219-19.el7_2.13.x86_64 2/10 Updating : systemd-sysv-219-19.el7_2.13.x86_64 3/10 Updating : systemd-python-219-19.el7_2.13.x86_64 4/10 Updating : libgudev1-219-19.el7_2.13.x86_64 5/10 Cleanup : systemd-sysv-219-19.el7_2.4.x86_64 6/10 Cleanup : systemd-python-219-19.el7_2.4.x86_64 7/10 Cleanup : systemd-219-19.el7_2.4.x86_64 8/10 Cleanup : libgudev1-219-19.el7_2.4.x86_64 9/10 Cleanup : systemd-libs-219-19.el7_2.4.x86_64 10/10 Verifying : systemd-libs-219-19.el7_2.13.x86_64 1/10 Verifying : systemd-sysv-219-19.el7_2.13.x86_64 2/10 Verifying : systemd-219-19.el7_2.13.x86_64 3/10 Verifying : systemd-python-219-19.el7_2.13.x86_64 4/10 Verifying : libgudev1-219-19.el7_2.13.x86_64 5/10 Verifying : systemd-libs-219-19.el7_2.4.x86_64 6/10 Verifying : systemd-sysv-219-19.el7_2.4.x86_64 7/10 Verifying : systemd-219-19.el7_2.4.x86_64 8/10 Verifying : libgudev1-219-19.el7_2.4.x86_64 9/10 Verifying : systemd-python-219-19.el7_2.4.x86_64 10/10 Updated: systemd.x86_64 0:219-19.el7_2.13 Dependency Updated: libgudev1.x86_64 0:219-19.el7_2.13 systemd-libs.x86_64 0:219-19.el7_2.13 systemd-python.x86_64 0:219-19.el7_2.13 systemd-sysv.x86_64 0:219-19.el7_2.13 Complete! [root@RESCUE-pirax-test dev]# su adam [adam@RESCUE-pirax-test dev]$ ssh root@localhost cannot read from /dev/urandom, No such file or directory [adam@RESCUE-pirax-test dev]$ /sbin/ Display all 526 possibilities? (y or n) [adam@RESCUE-pirax-test dev]$ /sbin/MAKEDEV std bash: /sbin/MAKEDEV: No such file or directory [adam@RESCUE-pirax-test dev]$ mknod /dev/random c 1 9 mknod: ‘/dev/random’: Permission denied [adam@RESCUE-pirax-test dev]$ exit exit [root@RESCUE-pirax-test dev]# mknod /dev/random c 1 9 [root@RESCUE-pirax-test dev]# su adam [adam@RESCUE-pirax-test dev]$ ssh root@localhost cannot read from /dev/urandom, No such file or directory [adam@RESCUE-pirax-test dev]$ exit exit [root@RESCUE-pirax-test dev]# mknod /dev/urandom c 1 9 [root@RESCUE-pirax-test dev]# su adam [adam@RESCUE-pirax-test dev]$ ssh root@localhost Host key verification failed. [adam@RESCUE-pirax-test dev]$ exit
Dec 7 09:23:55 pirax-test login: FAILED LOGIN 1 FROM tty1 FOR root, Authentication failure Dec 7 09:32:00 pirax-test polkitd[1031]: Loading rules from directory /etc/polkit-1/rules.d Dec 7 09:32:00 pirax-test polkitd[1031]: Loading rules from directory /usr/share/polkit-1/rules.d Dec 7 09:32:00 pirax-test polkitd[1031]: Finished loading, compiling and executing 2 rules Dec 7 09:32:00 pirax-test polkitd[1031]: Acquired the name org.freedesktop.PolicyKit1 on the system bus Dec 7 09:32:10 pirax-test sshd[1375]: Server listening on 0.0.0.0 port 666. Dec 7 09:32:10 pirax-test sshd[1375]: Server listening on :: port 666. Dec 7 09:32:24 pirax-test unix_chkpwd[2692]: password check failed for user (root) Dec 7 09:32:24 pirax-test login: pam_unix(login:auth): authentication failure; logname=LOGIN uid=0 euid=0 tty=tty1 ruser= rhost= user=root Dec 7 09:32:24 pirax-test login: pam_succeed_if(login:auth): requirement "uid >= 1000" not met by user "root" Dec 7 09:32:27 pirax-test login: FAILED LOGIN 1 FROM tty1 FOR root, Authentication failure Dec 7 09:32:32 pirax-test unix_chkpwd[2694]: password check failed for user (root) Dec 7 09:32:32 pirax-test login: pam_succeed_if(login:auth): requirement "uid >= 1000" not met by user "root" Dec 7 09:32:34 pirax-test login: FAILED LOGIN 2 FROM tty1 FOR root, Authentication failure Dec 7 09:32:39 pirax-test unix_chkpwd[2696]: password check failed for user (root) Dec 7 09:32:39 pirax-test login: pam_succeed_if(login:auth): requirement "uid >= 1000" not met by user "root" Dec 7 09:32:41 pirax-test login: FAILED LOGIN SESSION FROM tty1 FOR root, Authentication failure Dec 7 09:32:41 pirax-test login: PAM 2 more authentication failures; logname=LOGIN uid=0 euid=0 tty=tty1 ruser= rhost= user=root Dec 7 10:05:17 pirax-test polkitd[1029]: Loading rules from directory /etc/polkit-1/rules.d Dec 7 10:05:17 pirax-test polkitd[1029]: Loading rules from directory /usr/share/polkit-1/rules.d Dec 7 10:05:17 pirax-test polkitd[1029]: Finished loading, compiling and executing 2 rules Dec 7 10:05:17 pirax-test polkitd[1029]: Acquired the name org.freedesktop.PolicyKit1 on the system bus Dec 7 10:05:27 pirax-test sshd[1375]: Server listening on 0.0.0.0 port 666. Dec 7 10:05:27 pirax-test sshd[1375]: Server listening on :: port 666. Dec 7 10:09:56 pirax-test unix_chkpwd[2791]: password check failed for user (root) Dec 7 10:09:56 pirax-test login: pam_unix(login:auth): authentication failure; logname=LOGIN uid=0 euid=0 tty=tty1 ruser= rhost= user=root Dec 7 10:09:56 pirax-test login: pam_succeed_if(login:auth): requirement "uid >= 1000" not met by user "root" Dec 7 10:09:58 pirax-test login: FAILED LOGIN 1 FROM tty1 FOR root, Authentication failure Dec 7 10:10:07 pirax-test unix_chkpwd[2802]: password check failed for user (root) Dec 7 10:10:07 pirax-test login: pam_succeed_if(login:auth): requirement "uid >= 1000" not met by user "root" Dec 7 10:10:09 pirax-test login: FAILED LOGIN 2 FROM tty1 FOR root, Authentication failure [root@RESCUE-pirax-test /]# adduser adam adduser: user 'adam' already exists [root@RESCUE-pirax-test /]# passwd adam Changing password for user adam. New password: Retype new password: passwd: all authentication tokens updated successfully. [root@RESCUE-pirax-test /]# passwd root Changing password for user root. New password: Retype new password: passwd: all authentication tokens updated successfully. [root@RESCUE-pirax-test /]# su adam [adam@RESCUE-pirax-test /]$ ssh adam@localhost Host key verification failed. [adam@RESCUE-pirax-test /]$ exit exit [root@RESCUE-pirax-test /]# vi /etc/ss ssh/ ssl/ [root@RESCUE-pirax-test /]# vi /etc/ss ssh/ ssl/ [root@RESCUE-pirax-test /]# vi /etc/ssh/ moduli ssh_config sshd_config ssh_host_ecdsa_key ssh_host_ecdsa_key.pub ssh_host_ed25519_key ssh_host_ed25519_key.pub ssh_host_rsa_key ssh_host_rsa_key.pub [root@RESCUE-pirax-test /]# vi /etc/ssh/sshd_config [root@RESCUE-pirax-test /]# su adam [adam@RESCUE-pirax-test /]$ ssh adam@localhost Host key verification failed. [adam@RESCUE-pirax-test /]$ exit exit [root@RESCUE-pirax-test /]# mknod -m 666 /dev/tty c 5 0 [root@RESCUE-pirax-test /]# su adam [adam@RESCUE-pirax-test /]$ ssh adam@localhost The authenticity of host 'localhost (::1)' can't be established. ECDSA key fingerprint is e1:0c:0f:95:94:55:84:00:0d:e9:36:42:1d:6d:9e:0c. Are you sure you want to continue connecting (yes/no)? yes Warning: Permanently added 'localhost' (ECDSA) to the list of known hosts. adam@localhost's password:
It’s still kind of broken, but its getting less broken by the minute ;-D