结合使用加密/ ssh和代理身份验证，同时回退到加密密钥身份验证

I am implementing an SSH client which will use agent authentication (if available) and fallback to public key authentication if that fails. I have found that I can do this using multiple signers like the following:

sshConfig := &ssh.ClientConfig{
    User:            getUsername(username, currentUser),
    HostKeyCallback: ssh.InsecureIgnoreHostKey(),
    Auth: []ssh.AuthMethod{
        ssh.PublicKeysCallback(getSigners),
    },
}

func getSigners() ([]ssh.Signer, error) {
    signers := make([]ssh.Signer, 0)
    currentUser, _ := user.Current()
    if os.Getenv("SSH_AUTH_SOCK") != "" {
        sshAgent, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
        if err == nil {
            agentSigners, _ := agent.NewClient(sshAgent).Signers()
            signers = append(signers, agentSigners...)
        }
    }

    // default to id_rsa
    keyPath := path.Join(path.Join(currentUser.HomeDir, ".ssh/id_rsa"))

    buffer, errI := ioutil.ReadFile(keyPath)
    if errI != nil {
        fmt.Println(errI)
        return signers, errI
    }

    block, _ := pem.Decode(buffer)

    var key ssh.Signer
    if strings.Contains(block.Headers["Proc-Type"], "ENCRYPTED") {
        fmt.Print("SSH Passphrase: ")
        bytePassword, _ := terminal.ReadPassword(int(syscall.Stdin))
        key, _ = ssh.ParsePrivateKeyWithPassphrase(buffer, bytePassword)
    } else {
        key, _ = ssh.ParsePrivateKey(buffer)
    }

    signers = append(signers, key)
    return signers, nil
}

The issue with this method is that it will always prompt for a passphrase, since the passphrase prompt occurs before any authentication attempts have been made. Is it possible to delay when the public key gets decrypted so that agent authentication can be attempted before the user is asked to enter a passphrase?