Decorating a rollover toolbar button with JLayer paints its border

Question

I'm attempting to wrap a JButton with a JLayer to add some effects/functionality to it. When I replace the button with the wrapper in a toolbar, it paints a button border for some reason. The toolbar has rollover set to true.

Why is this happening and how can I prevent it?

import java.awt.AWTEvent;
import java.awt.BorderLayout;
import java.awt.Graphics;
import java.awt.event.MouseEvent;
import javax.swing.JButton;
import javax.swing.JComponent;
import javax.swing.JFrame;
import javax.swing.JLayer;
import javax.swing.JToolBar;
import javax.swing.SwingUtilities;
import javax.swing.UIManager;
import javax.swing.UnsupportedLookAndFeelException;
import javax.swing.plaf.LayerUI;

public class JLayerButton extends JFrame {

    private JToolBar toolbar;
    private JButton button1;
    private JButton button2;
    private JButton button3;

    public JLayerButton() {
        setDefaultCloseOperation(EXIT_ON_CLOSE);
        setTitle("JLayer Button");
        setLayout(new BorderLayout());

        toolbar = new JToolBar();
        toolbar.setFloatable(false);
        toolbar.setRollover(true);
        getContentPane().add(toolbar, BorderLayout.NORTH);

        button1 = new JButton("One");
        button1.setFocusable(false);
        toolbar.add(button1);
        button2 = new JButton("Two");
        button2.setFocusable(false);
        toolbar.add(button2);
        button3 = new JButton("Three");
        button3.setFocusable(false);
        toolbar.add(button3);

        // wrap button2 in JLayer
        int componentIndex = toolbar.getComponentIndex(button2);
        JButtonLayerUI layerUI = new JButtonLayerUI();
        JLayer<JButton> layer = new JLayer<JButton>(button2, layerUI);
        layer.setLayerEventMask(AWTEvent.MOUSE_EVENT_MASK | AWTEvent.MOUSE_MOTION_EVENT_MASK);
        layer.setFocusable(false);
        toolbar.add(layer, componentIndex);

        setSize(300, 200);
        setLocationRelativeTo(null);
    }

    public static void main(String[] args) 
            throws ClassNotFoundException, InstantiationException, 
            IllegalAccessException, UnsupportedLookAndFeelException {
        UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName());

        SwingUtilities.invokeLater(new Runnable() {
            @Override
            public void run() {
                new JLayerButton().setVisible(true);
            }
        });
    }

    private static class JButtonLayerUI extends LayerUI<JButton> {

        public JButtonLayerUI() {
        }

        @Override
        public void paint(Graphics g, JComponent c) {
            super.paint(g, c);
        }

        @Override
        protected void processMouseEvent(MouseEvent e, JLayer<? extends JButton> l) {
            onMouseEvent(l, e);
        }

        @Override
        protected void processMouseMotionEvent(MouseEvent e, JLayer<? extends JButton> l) {
            onMouseEvent(l, e);
        }

        private void onMouseEvent(JLayer<? extends JButton> l, MouseEvent e) {
            System.out.println(e);
        }        

    }
}

Answer 1

What you want to achieve doesn't seem possible with JLayer, since :

From the code of BasicToolBarUI, we can see those calls

installRolloverBorders is called to install borders on the JToolBar :

public void setRolloverBorders( boolean rollover ) {
        rolloverBorders = rollover;

        if ( rolloverBorders )  {
            installRolloverBorders( toolBar );
        } else  {
            installNonRolloverBorders( toolBar );
        }
    }

installRolloverBorders calls setBorderToRollover on each of the direct JComponent childs (non recursive) of the toolbar

protected void installRolloverBorders ( JComponent c )  {
        // Put rollover borders on buttons
        Component[] components = c.getComponents();

        for (Component component : components) {
            if (component instanceof JComponent) {
                ((JComponent) component).updateUI();
                setBorderToRollover(component);
            }
        }
    }

Finally, setBorderToRollover will only work on instances of AbstractButton, which your JLayer is not. So the original borders of your JLayer's buttons are not affected at all.

protected void setBorderToRollover(Component c) {
        if (c instanceof AbstractButton) {
            AbstractButton b = (AbstractButton)c;

            Border border = borderTable.get(b);
            if (border == null || border instanceof UIResource) {
                borderTable.put(b, b.getBorder());
            }

            // Only set the border if its the default border
            if (b.getBorder() instanceof UIResource) {
                b.setBorder(getRolloverBorder(b));
            }

            rolloverTable.put(b, b.isRolloverEnabled()?
                              Boolean.TRUE: Boolean.FALSE);
            b.setRolloverEnabled(true);
        }
    }

To sum it up, the rollover effect will only work on components meeting both of those conditions :

1. Be a direct child of the JToolBar
2. Be an instance of AbstractButton

You probably could roll your own version of BasicToolBarUI and override some methods to add recursivity for instance, see if it is worth the time.