frame.setResizeable(false);

public class Test extends JDialog {
public Test(JFrame frame, String str) {
    super(frame, str);
    addWindowListener(new WindowAdapter() {
        public void windowClosing(WindowEvent evt) {
            System.exit(0);
        }
    });
}

public static void main(String[] args) {
    try {
        Test myFrame = new Test(new JFrame(), "Removing maximize button");
        JPanel panel = new JPanel();
        panel.setSize(100, 100);
        myFrame.add(panel);
        myFrame.setSize(100, 100);
        myFrame.setVisible(true);
    } catch (IllegalArgumentException e) {
        System.exit(0);
    }
} }