public class RoundedLinearLayout extends LinearLayout {

private Paint drawPaint;
private Paint roundPaint;

private int mCornerRadius = 100;

private RectF bounds;

@TargetApi(Build.VERSION_CODES.HONEYCOMB)
public RoundedLinearLayout(Context context, AttributeSet attrs, int defStyle) {
    super(context, attrs, defStyle);
    onInit();
}

public RoundedLinearLayout(Context context, AttributeSet attrs) {
    super(context, attrs);
    onInit();
}

public RoundedLinearLayout(Context context) {
    super(context);
    onInit();
}

protected void onInit() {
    drawPaint = new Paint(Paint.ANTI_ALIAS_FLAG);
    drawPaint.setColor(0xffffffff);
    drawPaint.setXfermode(new PorterDuffXfermode(PorterDuff.Mode.SRC_IN));

    roundPaint = new Paint(Paint.ANTI_ALIAS_FLAG);
    roundPaint.setColor(0xffffffff);

    setWillNotDraw(false);
}

@Override
protected void onSizeChanged(int w, int h, int oldw, int oldh) {
    super.onSizeChanged(w, h, oldw, oldh);
    if (w != oldw && h != oldh) {
        bounds = new RectF(0, 0, w, h);
    }
}

@Override
protected void dispatchDraw(Canvas canvas) {
    Bitmap bitmap = Bitmap.createBitmap((int) bounds.width(), (int) bounds.height(), Bitmap.Config.ARGB_8888);
    Canvas c = new Canvas(bitmap);
    super.dispatchDraw(c);

    BitmapShader shader = new BitmapShader(bitmap, Shader.TileMode.CLAMP, Shader.TileMode.CLAMP);

    Paint paint = new Paint();
    paint.setAntiAlias(true);
    paint.setShader(shader);

    canvas.drawRoundRect(bounds, mCornerRadius, mCornerRadius, paint);
}
}

LayoutInflater  factory = LayoutInflater.from(getActivity());
            AlertDialog alert = new AlertDialog.Builder(getActivity());

        Dialog  dialog = new Dialog(getActivity());

            dialog.setContentView(your layout);

            dialog.getWindow().setBackgroundDrawable(
                    new ColorDrawable(android.graphics.Color.TRANSPARENT));

myDialog
    .getWindow()
    .setBackgroundDrawable(new ColorDrawable(Color.argb(0,0,0,0)));

<shape xmlns:android="http://schemas.android.com/apk/res/android">
    ...
    <corners
        android:radius="3dp" />
    ...
</shape>

MyDialog.getWindow().setBackgroundDrawable(new ColorDrawable(android.graphics.Color.TRANSPARENT));

Dialog MyDialog= new Dialog(MyActivity.this);
MyDialog.requestWindowFeature(Window.FEATURE_NO_TITLE);
MyDialog.setContentView(R.layout.my_custom_dialog);
MyDialog.getWindow().setBackgroundDrawable(new ColorDrawable(android.graphics.Color.TRANSPARENT));
MyDialog.show();