Freeform mode (Android 7.0 - 8.1)

// From android.app.ActivityManager.StackId
final int FREEFORM_WORKSPACE_STACK_ID = 2;

ActivityOptions options = ActivityOptions.makeBasic();

// Note: Android 8.x broke setLaunchBounds(), and as a result,
// doesn't actually apply the bounds specified in the Rect.
// The app will still launch in a freeform window, though.
options.setLaunchBounds(new Rect(0, 0, 500, 500));

try {
    Method method = ActivityOptions.class.getMethod("setLaunchStackId", int.class);
    method.invoke(options, FREEFORM_WORKSPACE_STACK_ID);
} catch (Exception e) { /* handle exception */ }

// Now, call startActivity(intent, options.toBundle());