# class XLSummary(BaseTextDataset):
#     def __init__(self, tokenizer: BaseTokenizer, max_seq_length: int = 128, dataset_url: Optional[str] = None, save_dir: str = "./datasets", filename: str = "xlsum-en-300k.json"):
#         super().__init__(tokenizer, max_seq_length, dataset_url)

#     def load_data(self, dataset_url: Optional[str] = None, save_dir: str = "./datasets", filename: str = "xlsum-en-300k.json"):
#         import base64

#         N_STEPS_PER_TQDM_UPDATE = 10
#         steps_taken = 0

#         """
#         import base64
#         strb = bytes("Hi", 'utf-8')
#         encoded = base64.b64encode(strb).decode('utf-8')
#         print(encoded)
#         """

#         if not os.path.isfile(dataset_url):
#             raise Exception(f"`{dataset_url}` does not exist!")

#         with open(dataset_url, 'r') as file:
#             data = json.load(file)

#         # A running list of previous question/answer pairs to augment the current entry.
#         chat = ChatHistory()

#         tqdm_bar = tqdm(total=len(data), desc="Loading", position=0, leave=True)
#         for i, entry in enumerate(data):
#             steps_taken += 1

#             if steps_taken % N_STEPS_PER_TQDM_UPDATE == 0:
#                 tqdm_bar.set_description(f"Processing {steps_taken}/{len(data)}")
#                 tqdm_bar.update(N_STEPS_PER_TQDM_UPDATE)

#             document = entry['text']
#             summary = entry['summary']

#             # Assistant: OK!
#             agent_confirmation = random.choice(self.CONFIRMATIONS)
#             agent_remark = random.choice(self.REMARKS)
#             # Newlines, breaks etc.
#             _whitespace1 = self.make_whitespace()
#             _whitespace2 = self.make_whitespace()

#             as_json = json.dumps({
#                 "summary": summary,
#             }, indent=4)

#             as_json_with_original = json.dumps({
#                 "original": document,
#                 "summary": summary,
#             }, indent=4)

#             as_json_with_b64 = json.dumps({
#                 "summary": summary,
#                 "b64": base64.b64encode(bytes(summary, 'utf-8')).decode('utf-8'),
#             }, indent=4)

#             qa = [
#                 {
#                     "question": f"{document}{_whitespace2}Summarise^",
#                     "response": f"{agent_confirmation} Here's a summary of the document you provided:\n\n{summary}\n\n{agent_remark}",
#                 },
#                 {
#                     "question": f"Please summarise the following document:{_whitespace1}{document}{_whitespace2}Make sure to output nothing else but the summary.",
#                     "response": f"{summary}",
#                 },
#                 {
#                     "question": f"ONLY summarise the document below.{_whitespace1}{document}{_whitespace2}",
#                     "response": f"{summary}",
#                 },
#                 {
#                     "question": f"Can you summarise the document below?{_whitespace1}{document}{_whitespace2}",
#                     "response": f"{agent_confirmation} I've provided a brief explanation of the text you supplied above:\n\n{summary}\n\n{agent_remark}",
#                 },
#                 {
#                     "question": f"{_whitespace1}{document}{_whitespace2}I want you to summarise the document provided, however I want you to output it in JSON format.",
#                     "response": f"{agent_confirmation} I'll provide the summary as a key in the JSON payload below:\n```json\n{as_json}\n```\n",
#                 },
#                 {
#                     "question": f"Please can you summarise this document:{_whitespace1}{document}{_whitespace2}Are you able to output it as JSON with a 'b64' parameter with the base64 encoding?",
#                     "response": f"{agent_confirmation} Here's a summary of the text you provided with the base64 encoding attached.\n```json\n{as_json_with_b64}\n```\n{agent_remark}",
#                 },
#                 {
#                     "question": f"<Document>{_whitespace1}{document}{_whitespace2}</Document>\n\nSummarize document above as a JSON payload, nothing else.",
#                     "response": f"```json\n{as_json}\n```",
#                 },
#                 {
#                     "question": f"<Document>{_whitespace1}{document}{_whitespace2}</Document>\n\nSummarize document above as a JSON payload and include the 'original' text in the payload.",
#                     "response": f"Sure, here's the JSON data as requested:\n\n```json\n{as_json_with_original}\n```\n{agent_remark}",
#                 },
#             ]

#             selection = random.choice(qa)

#             question = selection['question']
#             response = selection['response']

#             chat.add_message(role="Human", content=question)
#             chat.add_message(role="Assistant", content=response)
#             encoded = self.tokenizer.encode(chat.transcript())

#             self.dataset.append(encoded)
#         return self.dataset