1

I have a set of RPi4s (7 of them) running Ubuntu 20.04. One of them is the gui, from where I run commands. There are 3 managers and 3 workers (/etc/hosts is properly set up on all of them).

Initially, they all have a user with the same name and password in all of them.

I want to set up SSH in a single script run once in the gui (gui0), so that then I have paswordless SSH.

The gui* machines should have acces to all; manager* machines should have access to managers and workers; and workers should only be able to access other workers.

The reason for that is that I'm going to set up storage with GlusterFS (and a few other things), so the machines need to be able to talk between them.

I developed a script, which more or less works: Depending on the time that I sleep between some of the commands, there are more keys that are actually copied, but I'm already sleeping huge times, and still don't reach 100%.

Script:

#!/bin/bash

################################################################################
##  source                                    ##
################################################################################
source  lib/libalx/sh/sysexits.sh;  ## This provides EX_USAGE=64


################################################################################
##  definitions                               ##
################################################################################
ARGC=0;

guis="gui0";
managers="manager0 manager1 manager2";
workers="worker0 worker1 worker2";
all_machines="${guis} ${managers} ${workers}";
gui_accessible_machines="${all_machines}";
manager_accessible_machines="${managers} ${workers}";
worker_accessible_machines="${workers}";


################################################################################
##  functions                                 ##
################################################################################
## XXX: Pair calls to this function with "unset SSHPASS"!!!
function read_ssh_password()
{

    echo "This script will set up keyless ssh."
    echo "After this script, ssh will not accept passwords again."
    echo "Enter the current password for ssh connections."

    read -s -p "Password to use: " SSHPASS;
    echo;
    export SSHPASS;
}

function create_ssh_keys()
{

    for remote in ${all_machines}; do
        echo "  SSH-KEYGEN  ${remote};"
        sshpass -e ssh ${remote} "
            ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_rsa;
        ";
    done
}

function distribute_ssh_keys_to()
{
    local   accessible_machines="$1";
#   local   ssh_opts="-o PreferredAuthentications=keyboard-interactive";
#   ssh_opts="${ssh_opts} -o PubkeyAuthentication=no";

    for remote in ${accessible_machines}; do
        echo "  SSH-COPY-ID $(cat /etc/hostname)    ${remote};"
        sshpass -e ssh-copy-id -i ~/.ssh/id_rsa.pub ${remote}   \
        2>&1 | grep -e WARNING -e ERROR -e added;
        sleep 60;
    done
}

function distribute_ssh_keys_from()
{
#   ssh_opts="-o PreferredAuthentications=keyboard-interactive";
#   ssh_opts="${ssh_opts} -o PubkeyAuthentication=no";
    local   machines="$1";
    local   accessible_machines="$2";

    for remote in ${machines}; do
        sshpass -e ssh ${remote} "
            $(declare -fg);
            export SSHPASS=${SSHPASS};
            distribute_ssh_keys_to  \"${accessible_machines}\";
            unset SSHPASS;
        ";
        sleep 300;
    done
    sleep 300;
}

function distribute_ssh_keys()
{

    distribute_ssh_keys_from "${guis}" "${gui_accessible_machines}";
    distribute_ssh_keys_from "${managers}" "${manager_accessible_machines}";
    distribute_ssh_keys_from "${workers}" "${worker_accessible_machines}";

    for remote in ${all_machines}; do
        ssh ${remote} "
            $(declare -fg);
            secure_ssh;
        ";
        sleep 60;
    done
}

function secure_ssh()
{

    :; ## TODO
}

function create_distribute_ssh_keys()
{

    read_ssh_password;

    create_ssh_keys;
    sleep 300;
    distribute_ssh_keys;

    unset SSHPASS;
}


################################################################################
##  main                                      ##
################################################################################
function main()
{

    create_distribute_ssh_keys;
}


################################################################################
##  run                                   ##
################################################################################
argc=$#;
if [ ${argc} -ne ${ARGC} ]; then
    echo    "Illegal number of parameters (Requires ${ARGC})";
    exit    ${EX_USAGE};
fi

main;

Output:

ubuntu@gui0:~$ ./bin/setup_ssh.sh 
This script will set up keyless ssh.
After this script, ssh will not accept passwords again.
Enter the current password for ssh connections.
Password to use: 
    SSH-KEYGEN  gui0;
Generating public/private rsa key pair.
/home/ubuntu/.ssh/id_rsa already exists.
Overwrite (y/n)? n
    SSH-KEYGEN  manager0;
Generating public/private rsa key pair.
/home/ubuntu/.ssh/id_rsa already exists.
Overwrite (y/n)? n
    SSH-KEYGEN  manager1;
Generating public/private rsa key pair.
/home/ubuntu/.ssh/id_rsa already exists.
Overwrite (y/n)? n
    SSH-KEYGEN  manager2;
Generating public/private rsa key pair.
/home/ubuntu/.ssh/id_rsa already exists.
Overwrite (y/n)? n
    SSH-KEYGEN  worker0;
Generating public/private rsa key pair.
/home/ubuntu/.ssh/id_rsa already exists.
Overwrite (y/n)? n
    SSH-KEYGEN  worker1;
Generating public/private rsa key pair.
/home/ubuntu/.ssh/id_rsa already exists.
Overwrite (y/n)? n
    SSH-KEYGEN  worker2;
Generating public/private rsa key pair.
/home/ubuntu/.ssh/id_rsa already exists.
Overwrite (y/n)? n
    SSH-COPY-ID gui0    gui0;
/usr/bin/ssh-copy-id: WARNING: All keys were skipped because they already exist on the remote system.
    SSH-COPY-ID gui0    manager0;
/usr/bin/ssh-copy-id: WARNING: All keys were skipped because they already exist on the remote system.
    SSH-COPY-ID gui0    manager1;
/usr/bin/ssh-copy-id: WARNING: All keys were skipped because they already exist on the remote system.
    SSH-COPY-ID gui0    manager2;
/usr/bin/ssh-copy-id: WARNING: All keys were skipped because they already exist on the remote system.
    SSH-COPY-ID gui0    worker0;
/usr/bin/ssh-copy-id: WARNING: All keys were skipped because they already exist on the remote system.
    SSH-COPY-ID gui0    worker1;
/usr/bin/ssh-copy-id: WARNING: All keys were skipped because they already exist on the remote system.
    SSH-COPY-ID gui0    worker2;
/usr/bin/ssh-copy-id: WARNING: All keys were skipped because they already exist on the remote system.

    SSH-COPY-ID manager0    manager0;
/usr/bin/ssh-copy-id: WARNING: All keys were skipped because they already exist on the remote system.
    SSH-COPY-ID manager0    manager1;
/usr/bin/ssh-copy-id: WARNING: All keys were skipped because they already exist on the remote system.
    SSH-COPY-ID manager0    manager2;
/usr/bin/ssh-copy-id: WARNING: All keys were skipped because they already exist on the remote system.
    SSH-COPY-ID manager0    worker0;
Number of key(s) added: 1
    SSH-COPY-ID manager0    worker1;
    SSH-COPY-ID manager0    worker2;
/usr/bin/ssh-copy-id: WARNING: All keys were skipped because they already exist on the remote system.
    SSH-COPY-ID manager1    manager0;
    SSH-COPY-ID manager1    manager1;
    SSH-COPY-ID manager1    manager2;
    SSH-COPY-ID manager1    worker0;
    SSH-COPY-ID manager1    worker1;
    SSH-COPY-ID manager1    worker2;
    SSH-COPY-ID manager2    manager0;
    SSH-COPY-ID manager2    manager1;
    SSH-COPY-ID manager2    manager2;
    SSH-COPY-ID manager2    worker0;
    SSH-COPY-ID manager2    worker1;
    SSH-COPY-ID manager2    worker2;
    SSH-COPY-ID worker0 worker0;
    SSH-COPY-ID worker0 worker1;
    SSH-COPY-ID worker0 worker2;
    SSH-COPY-ID worker1 worker0;
    SSH-COPY-ID worker1 worker1;
    SSH-COPY-ID worker1 worker2;
    SSH-COPY-ID worker2 worker0;
    SSH-COPY-ID worker2 worker1;
    SSH-COPY-ID worker2 worker2;

I expect to receive always either a line saying the number of keys added, or some ERROR or WARNING. But there are cases where I only receive INFO lines (which I discard as noise with grep). As you can see, the first few work (it shows a WARNING because it's not the first time I run it, so the keys were already installed on a previous run, but that's fine), then some start failing, and then all of them fail.

If I reduce the sleep time, it starts failing earlier.

As an example of what happens without grep:

    SSH-COPY-ID manager0;   worker0;
/usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/home/ubuntu/.ssh/id_rsa.pub"
/usr/bin/ssh-copy-id: INFO: attempting to log in with the new key(s), to filter out any that are already installed
/usr/bin/ssh-copy-id: INFO: 1 key(s) remain to be installed -- if you are prompted now it is to install the new keys

Number of key(s) added: 1

Now try logging into the machine, with:   "ssh 'worker0'"
and check to make sure that only the key(s) you wanted were added.

    SSH-COPY-ID manager0;   worker1;
/usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/home/ubuntu/.ssh/id_rsa.pub"
    SSH-COPY-ID manager0;   worker2;
/usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/home/ubuntu/.ssh/id_rsa.pub"
/usr/bin/ssh-copy-id: INFO: attempting to log in with the new key(s), to filter out any that are already installed
/usr/bin/ssh-copy-id: INFO: 1 key(s) remain to be installed -- if you are prompted now it is to install the new keys

Number of key(s) added: 1

Now try logging into the machine, with:   "ssh 'worker2'"
and check to make sure that only the key(s) you wanted were added.

Why is it failing so erratically?

5
  • 1
    bash. Yes, I have the shebang, which I forgot to copy :) I'll add it to the post
    – alx
    Nov 27 '20 at 15:44
  • I'm trying to understand what you want to achieve by running this script. You want to be able to ssh from gui0 to each of the managers and workers, ok. Do you want to be able to ssh from each of the managers to their respective worker, or back to gui0, or...? If not, why are you running ssh-keygen on any of the remote systems?
    – roaima
    Nov 27 '20 at 15:47
  • gui0 should be able to ssh into any system. manager0, should be able to ssh into manager0, manager1, manager2, worker0, worker1, worker2 (and other managers should be able to). worker0 (and all other workers) should be able to ssh into any other worker (or itself). So yes, I want to ssh from some non-gui to some other non-gui system.
    – alx
    Nov 27 '20 at 15:49
  • And why?: I'm about to set up glusterfs (and a few other things), so the machines need to be able to talk between them.
    – alx
    Nov 27 '20 at 15:55
  • @roaima I added that to the post :)
    – alx
    Nov 27 '20 at 17:39
2

I can't see why you've got the long sleep statements spread around, and I'm not convinced I've traced your program's flow correctly. However, looking at the requirements, you need

The gui* machines should have access to all; manager* machines should have access to managers and workers; and workers should only be able to access other workers

So

  • GUI → GUI Managers Workers
  • Managers → Managers Workers
  • Workers → Workers

Something like this then. You'll want to add your own progress updates and exit status, I'm sure, but this barebones should get you running.

#!/bin/bash
#
guis=(gui0)
managers=(manager0 manager1 manager2)
workers=(worker0 worker1 worker2)

# Grab the password
#
IFS= read -rsp "Master password: " sshpass && echo


# First, GUI to everything
#
if [[ ! -f "$HOME/.ssh/id_rsa" ]] || [[ ! -f "$HOME/.ssh/id_rsa.pub" ]]
then
    # Start clean
    rm -f "$HOME/.ssh/id_rsa" "$HOME/.ssh/id_rsa.pub"
    ssh-keygen -t rsa -b 4096 -f "$HOME/.ssh/id_rsa" -P ""
fi

for dst in "${guis[@]}" "${managers[@]}" "${workers[@]}"
do
    # Using the password we entered at the beginning, copy the keys everywhere
    SSHPASS=$sshpass sshpass -ev ssh-copy-id -o StrictHostKeyChecking=no -i "$HOME/.ssh/id_rsa" "$dst"
done


# Now generate a key on each host in turn
#
for dst in "${managers[@]}" "${workers[@]}"
do
    # Ensure the target is clean and then generate a new key
    ssh -n "$dst" 'rm -f .ssh/id_rsa .ssh/id_rsa.pub'
    ssh -n "$dst" 'ssh-keygen -t rsa -b 4096 -f .ssh/id_rsa -P ""'
done


# Grab each host's key pair
#
for src in "${managers[@]}" "${workers[@]}"
do
    scp -p "$src:.ssh/id_rsa" "$HOME/.ssh/id_rsa.$src"
    scp -p "$src:.ssh/id_rsa.pub" "$HOME/.ssh/id_rsa.$src.pub"
done


# Push each Manager key out to the Managers and Workers
#
for src in "${managers[@]}"
do
    for dst in "${managers[@]}" "${workers[@]}"
    do
        ssh-copy-id -i "$HOME/.ssh/id_rsa.$src" "$dst"
    done
done


# Push each Worker key out to the Workers
#
for src in "${workers[@]}"
do
    for dst in "${workers[@]}"
    do
        ssh-copy-id -i "$HOME/.ssh/id_rsa.$src" "$dst"
    done
done


# Now fix up the "authenticity of host" warnings by connecting everywhere
#
for src in "${managers[@]}"
do
    for dst in "${managers[@]}" "${workers[@]}"
    do
        ssh -n "$src" ssh -n -o StrictHostKeyChecking=no "$dst" id >/dev/null
    done
done

for src in "${workers[@]}"
do
    for dst in "${workers[@]}"
    do
        ssh -n "$src" ssh -n -o StrictHostKeyChecking=no "$dst" id >/dev/null
    done
done


# Delete the unwanted key pairs from this host
#
for src in "${managers[@]}" "${workers[@]}"
do
    rm -f "$HOME/.ssh/id_rsa.$src" "$HOME/.ssh/id_rsa.$src.pub"
done


# All done
#
exit 0

Notice that everything is controlled from the client (gui0), and during the process none of the Managers or Workers initiates a copy of any file to any other machine.

10
  • Thanks! I'll have a look at it in a moment. I'm testing something: When I added delays and when I increased them later, I saw improvements, but it was just by luck: in between tests I had connected manually to some of the machines. I think the problem is that SSH is probably asking me if I want a new host to be added to known_hosts. So I've deleted all known_hosts, and as I expected, none of them works now. So I'll try to fix that. :)
    – alx
    Nov 27 '20 at 19:36
  • 1
    Oh. If that's the issue then ssh -o StrictHostKeyChecking=no is what you need
    – roaima
    Nov 27 '20 at 19:37
  • I tried 'accept-new' right now, which didn't work. 'no' did work :)
    – alx
    Nov 27 '20 at 19:40
  • However, your code looks cleaner! I'll still take some ideas from it ;) Thanks a lot!
    – alx
    Nov 27 '20 at 19:40
  • 1
    @CacahueteFrito no it's intentional. The IFS variable is reset to empty for just the read command. That and the -r flag stops read from trying to post-process your password input (otherwise things like leading spaces will get removed, backslashes will be processed rather than treated literally, etc.). Try read p; echo "p=$p." and then enter {space} a. Then repeat with IFS= read -r p; echo "p=$p."
    – roaima
    Nov 28 '20 at 10:01
0

The main reason was that SSH didn't know the hosts (~/.ssh/known_hosts), so it prompted to add the new hosts to the file. Of course, the script couldn't react to that.

The solution was easy, as @roaima noted in a comment to his answer: ssh ... -o StrictHostKeyChecking=no

My script had a few other issues, which I have fixed (I also improved a few things thanks to @roaima's code).

For anyone who may need to do this in the future, here's my working script:

#!/bin/bash
set -Eeo pipefail
################################################################################
##
## Set up ssh network
## ==================
##
##  Run this script from a guiX machine
##
################################################################################


################################################################################
##  source                                    ##
################################################################################
source  lib/libalx/sh/sysexits.sh;


################################################################################
##  definitions                               ##
################################################################################
ARGC=0;

guis=(gui0);
managers=(manager0 manager1 manager2);
workers=(worker0 worker1 worker2);
all_machines="${guis[*]} ${managers[*]} ${workers[*]}";
gui_accessible_machines="${all_machines}";
manager_accessible_machines="${managers[*]} ${workers[*]}";
worker_accessible_machines="${workers[*]}";

ssh_opts='-o StrictHostKeyChecking=no';


################################################################################
##  functions                                 ##
################################################################################
## XXX: Pair calls to this function with "unset SSHPASS"!!!
function read_ssh_password()
{

    echo "This script will set up keyless ssh."
    echo "After this script, ssh will not accept passwords again."
    echo "Enter the current password for ssh connections."

    read -s -p "Password to use: " SSHPASS;
    echo;
    export SSHPASS;
}

function create_ssh_keys()
{

    for remote in ${all_machines}; do
        echo "  SSH-KEYGEN  ${remote};"
        sshpass -e ssh ${ssh_opts} ${remote} "
            ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_rsa -P '' ||:;
        ";
    done
}

function distribute_ssh_keys_to()
{
    local   accessible_machines="$1";

    for remote in ${accessible_machines}; do
        echo "  SSH-COPY-ID $(cat /etc/hostname)    ${remote};"
        sshpass -e ssh-copy-id -i ~/.ssh/id_rsa.pub ${ssh_opts} \
                    ${remote}           \
        2>&1 | { grep -e WARNING -e ERROR ||:; };
    done
}

function distribute_ssh_keys_from()
{
    local   machines="$1";
    local   accessible_machines="$2";

    for remote in ${machines}; do
        sshpass -e ssh -n ${ssh_opts} ${remote} "
            set -Eeo pipefail
            $(declare -fg);
            export SSHPASS=\"${SSHPASS}\";
            ssh_opts=\"${ssh_opts}\";
            distribute_ssh_keys_to  \"${accessible_machines}\";
            unset SSHPASS;
        ";
    done
}

function distribute_ssh_keys()
{

    distribute_ssh_keys_from "${guis[*]}" "${gui_accessible_machines}";
    distribute_ssh_keys_from "${managers[*]}" "${manager_accessible_machines}";
    distribute_ssh_keys_from "${workers[*]}" "${worker_accessible_machines}";

    for remote in ${all_machines}; do
        ssh -n ${remote} "
            $(declare -fg);
            secure_ssh;
        ";
    done
}

function secure_ssh()
{

    :; ## TODO
}

function create_distribute_ssh_keys()
{

    read_ssh_password;
    create_ssh_keys;
    distribute_ssh_keys;

    unset SSHPASS;
}


################################################################################
##  main                                      ##
################################################################################
function main()
{

    create_distribute_ssh_keys;
}


################################################################################
##  run                                   ##
################################################################################
argc=$#;
if [ ${argc} -ne ${ARGC} ]; then
    echo    "Illegal number of parameters (Requires ${ARGC})";
    exit    ${EX_USAGE};
fi

main;


################################################################################
##  end of file                               ##
################################################################################

With the following output:

$ ./bin/setup_ssh.sh
This script will set up keyless ssh.
After this script, ssh will not accept passwords again.
Enter the current password for ssh connections.
Password to use: 
    SSH-KEYGEN  gui0;
Generating public/private rsa key pair.
/home/ubuntu/.ssh/id_rsa already exists.
Overwrite (y/n)? n
    SSH-KEYGEN  manager0;
Generating public/private rsa key pair.
/home/ubuntu/.ssh/id_rsa already exists.
Overwrite (y/n)? n
    SSH-KEYGEN  manager1;
Generating public/private rsa key pair.
/home/ubuntu/.ssh/id_rsa already exists.
Overwrite (y/n)? n
    SSH-KEYGEN  manager2;
Generating public/private rsa key pair.
/home/ubuntu/.ssh/id_rsa already exists.
Overwrite (y/n)? n
    SSH-KEYGEN  worker0;
Generating public/private rsa key pair.
/home/ubuntu/.ssh/id_rsa already exists.
Overwrite (y/n)? n
    SSH-KEYGEN  worker1;
Generating public/private rsa key pair.
/home/ubuntu/.ssh/id_rsa already exists.
Overwrite (y/n)? n
    SSH-KEYGEN  worker2;
Generating public/private rsa key pair.
/home/ubuntu/.ssh/id_rsa already exists.
Overwrite (y/n)? n
    SSH-COPY-ID gui0    gui0;
/usr/bin/ssh-copy-id: WARNING: All keys were skipped because they already exist on the remote system.
    SSH-COPY-ID gui0    manager0;
    SSH-COPY-ID gui0    manager1;
    SSH-COPY-ID gui0    manager2;
    SSH-COPY-ID gui0    worker0;
    SSH-COPY-ID gui0    worker1;
    SSH-COPY-ID gui0    worker2;
    SSH-COPY-ID manager0    manager0;
    SSH-COPY-ID manager0    manager1;
    SSH-COPY-ID manager0    manager2;
    SSH-COPY-ID manager0    worker0;
    SSH-COPY-ID manager0    worker1;
    SSH-COPY-ID manager0    worker2;
    SSH-COPY-ID manager1    manager0;
    SSH-COPY-ID manager1    manager1;
    SSH-COPY-ID manager1    manager2;
    SSH-COPY-ID manager1    worker0;
    SSH-COPY-ID manager1    worker1;
    SSH-COPY-ID manager1    worker2;
    SSH-COPY-ID manager2    manager0;
    SSH-COPY-ID manager2    manager1;
    SSH-COPY-ID manager2    manager2;
    SSH-COPY-ID manager2    worker0;
    SSH-COPY-ID manager2    worker1;
    SSH-COPY-ID manager2    worker2;
    SSH-COPY-ID worker0 worker0;
    SSH-COPY-ID worker0 worker1;
    SSH-COPY-ID worker0 worker2;
    SSH-COPY-ID worker1 worker0;
    SSH-COPY-ID worker1 worker1;
    SSH-COPY-ID worker1 worker2;
    SSH-COPY-ID worker2 worker0;
    SSH-COPY-ID worker2 worker1;
    SSH-COPY-ID worker2 worker2;

Testing:

ubuntu@gui0:~$ ssh manager0 'ssh worker2 cat /etc/hostname'
worker2
ubuntu@gui0:~$ ssh worker2 'ssh manager1 cat /etc/hostname'
Host key verification failed.

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.