procedure TForm1.FileOpenDialog1SelectionChange(Sender: TObject);
var
  Attrs: DWORD;
begin
  with FileOpenDialog1 do
  begin
    if Assigned(ShellItem) and Succeeded(ShellItem.GetAttributes(SFGAO_FOLDER, Attrs)) and
      (Attrs <> SFGAO_FOLDER) then
    begin
      // file is selected
    end
    else
    begin
      // folder is selected
    end;
  end;
end;