type
  TRichEditHelper = class helper for TRichEdit
  strict private
    class function EditStreamCallback(dwCookie: DWORD_PTR; pbBuff: PByte; CB: Longint; var pCB: Longint): Longint; stdcall; static;
  public
    function GetSelectionAsRTF: string; virtual;
  end;

implementation

{ TRichEditHelper }

class function TRichEditHelper.EditStreamCallback(dwCookie: DWORD_PTR; pbBuff: PByte; CB: Longint; var pCB: Longint): Longint; stdcall;
begin
  // casting here to TStream makes the method more flexible because you can pass any kind of TStream implementation
  // and so to use this callback for other functions using EM_STREAMOUT message passing streams
  pCB := TStream(dwCookie).Write(pbBuff^, CB);
  Result := IfThen(CB = pCB, 0, 1);
end;

function TRichEditHelper.GetSelectionAsRTF: string;
var
  EditStream: TEditStream;
  StringStream: TStringStream;
begin
  StringStream := TStringStream.Create;
  try
    EditStream.dwError := 0;
    EditStream.dwCookie := DWORD_PTR(StringStream);
    EditStream.pfnCallback := EditStreamCallback;
    Perform(EM_STREAMOUT, SF_RTF or SFF_SELECTION, LPARAM(@EditStream));
    Result := StringStream.DataString;
  finally
    StringStream.Free;
  end;
end;

// your TRichEdit instances gets the GetRTF helper method
procedure TForm1.Button1Click(Sender: TObject);
begin
  ShowMessage(RichEdit1.GetSelectionAsRTF);
end;

// another way of abstraction
type
  TRichEditHelper = class helper for TRichEdit
  strict private
    function TryGetStream(Stream: TStream; Flags: DWORD): Boolean; inline;
    class function EditStreamCallback(dwCookie: DWORD_PTR; pbBuff: PByte; CB: Longint; var pCB: Longint): Longint; stdcall; static;
  strict protected
    function GetContentAsRTF: string;
    function GetSelectionAsRTF: string;
  public
    property ContentAsRTF: string read GetContentAsRTF;
    property SelectionAsRTF: string read GetSelectionAsRTF;
  end;

implementation

{ TRichEditHelper }

class function TRichEditHelper.EditStreamCallback(dwCookie: DWORD_PTR; pbBuff: PByte; CB: Longint; var pCB: Longint): Longint; stdcall;
begin
  pcb := TStream(dwCookie).Write(pbBuff^, CB);
  Result := IfThen(CB = pcb, 0, 1);
end;

function TRichEditHelper.TryGetStream(Stream: TStream; Flags: DWORD): Boolean;
var
  EditStream: TEditStream;
begin
  EditStream.dwCookie := DWORD_PTR(Stream);
  EditStream.dwError := 0;
  EditStream.pfnCallback := EditStreamCallback;
  Result := (Perform(EM_STREAMOUT, WPARAM(Flags), LPARAM(@EditStream)) > 0) and (EditStream.dwError = 0);
end;

function TRichEditHelper.GetContentAsRTF: string;
var
  Stream: TStringStream;
begin
  Result := '';
  Stream := TStringStream.Create;
  try
    if TryGetStream(Stream, SF_RTF) then
      Result := Stream.DataString;
  finally
    Stream.Free;
  end;
end;

function TRichEditHelper.GetSelectionAsRTF: string;
var
  Stream: TStringStream;
begin
  Result := '';
  Stream := TStringStream.Create;
  try
    if TryGetStream(Stream, SF_RTF or SFF_SELECTION) then
      Result := Stream.DataString;
  finally
    Stream.Free;
  end;
end;