SSH to an ubuntu machine using avahi

Question

I have an ubuntu box that I connect to using avahi. Connecting to that box works fine for all services (I regularly use AFP, SSH and SMB on it) but I've noticed that whenever I connect to it from a mac using SSH (and using the ".local" dns name provided by avahi - eg. ssh <servername>.local) SSH tries to connect using ipv6, which for some reason times out (after two minutes) then it tries ipv4 which connects immediately.

I'd like to avoid this timeout, as it's really annoying for me and other users - if SSH tried ipv4 first or if ssh over ipv6 worked then that would solve the problem. But so far I've been unable to get either to work (the best I've managed is to specify the -4 option to SSH to stop it from trying ipv6 at all).

I'm using Ubuntu 10.04. Any solution has to be on the server (not the client) as there are multiple clients connecting. A possible complication might be that my LAN is set up to allow link-local ipv6 addresses only, but I have other servers (using Mac OS) that I can SSH into using ipv6)

I suspect that the problem could be solved by either preventing avahi from broadcasting the ipv6 address, or by enabling ssh over ipv6, but so far as I can tell avahi is already configured not to broadcast the ipv6 address and sshd is configured to allow ipv6 connections!

Here's my /etc/avahi/avahi-daemon.conf (I don't think I've changed anything from the ubuntu defaults)

 [server]
 #host-name=foo
 #domain-name=local
 #browse-domains=0pointer.de, zeroconf.org
 use-ipv4=yes
 use-ipv6=no
 #allow-interfaces=eth0
 #deny-interfaces=eth1
 #check-response-ttl=no
 #use-iff-running=no
 #enable-dbus=yes
 #disallow-other-stacks=no
 #allow-point-to-point=no

 [wide-area]
 enable-wide-area=yes

 [publish]
 #disable-publishing=no
 #disable-user-service-publishing=no
 #add-service-cookie=no
 #publish-addresses=yes
 #publish-hinfo=yes
 #publish-workstation=yes
 #publish-domain=yes
 #publish-dns-servers=192.168.50.1, 192.168.50.2
 #publish-resolv-conf-dns-servers=yes
 #publish-aaaa-on-ipv4=yes
 #publish-a-on-ipv6=no

 [reflector]
 #enable-reflector=no
 #reflect-ipv=no

 [rlimits]
 #rlimit-as=
 rlimit-core=0
 rlimit-data=4194304
 rlimit-fsize=0
 rlimit-nofile=300
 rlimit-stack=4194304
 rlimit-nproc=3

and here's my sshd_config (mainly updated to only allow pub/private keys):

 # What ports, IPs and protocols we listen for
 Port 22
 # Use these options to restrict which interfaces/protocols sshd will bind to
 #ListenAddress ::
 #ListenAddress 0.0.0.0
 Protocol 2
 # HostKeys for protocol version 2
 HostKey /etc/ssh/ssh_host_rsa_key
 HostKey /etc/ssh/ssh_host_dsa_key
 #Privilege Separation is turned on for security
 UsePrivilegeSeparation yes

 # Lifetime and size of ephemeral version 1 server key
 KeyRegenerationInterval 3600
 ServerKeyBits 768

 # Logging
 SyslogFacility AUTH
 LogLevel INFO

 # Authentication:
 LoginGraceTime 180
 PermitRootLogin no
 StrictModes yes

 RSAAuthentication yes
 PubkeyAuthentication yes
 #AuthorizedKeysFile     %h/.ssh/authorized_keys

 # Don't read the user's ~/.rhosts and ~/.shosts files
 IgnoreRhosts yes
 # For this to work you will also need host keys in /etc/ssh_known_hosts
 RhostsRSAAuthentication no
 # similar for protocol version 2
 HostbasedAuthentication no
 # Uncomment if you don't trust ~/.ssh/known_hosts for RhostsRSAAuthentication
 #IgnoreUserKnownHosts yes

 # To enable empty passwords, change to yes (NOT RECOMMENDED)
 PermitEmptyPasswords no

 # Change to yes to enable challenge-response passwords (beware issues with
 # some PAM modules and threads)
 ChallengeResponseAuthentication no

 # Change to no to disable tunnelled clear text passwords
 PasswordAuthentication no
 AllowGroups sshusers

 # Kerberos options
 #KerberosAuthentication no
 #KerberosGetAFSToken no
 #KerberosOrLocalPasswd yes
 #KerberosTicketCleanup yes

 # GSSAPI options
 #GSSAPIAuthentication no
 #GSSAPICleanupCredentials yes

 X11Forwarding yes
 X11DisplayOffset 10
 PrintMotd no
 PrintLastLog yes
 TCPKeepAlive yes
 #UseLogin no

 MaxStartups 10:30:60
 #Banner /etc/issue.net

 # Allow client to pass locale environment variables
 AcceptEnv LANG LC_*

 Subsystem sftp /usr/lib/openssh/sftp-server

 UsePAM yes

Does anyone have any ideas that I can try, or has experienced anything similar?