clean_dataset.py

## (optional) Approximated dedup using sklearn
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import pandas as pd
import numpy as np

df = pd.read_json('input.json', encoding='utf-8')
df = df[['instruction', 'input', 'output']]
vectorizer = TfidfVectorizer().fit_transform(df['instruction'])
cosine_sim = cosine_similarity(vectorizer)
threshold = 0.8
to_remove = []

for i in range(cosine_sim.shape[0]):
    for j in range(i+1, cosine_sim.shape[1]):
        if cosine_sim[i, j] > threshold:
            to_remove.append(j)

df_deduplicated = df.drop(df.index[to_remove]) # Remove the marked questions
df_deduplicated.to_json('input_sklearn.json', force_ascii=False, indent=4, orient='records')




## fix grammar using 'wisdominanutshell/coedit-xxl-8bit' ≈20GB VRAM
import json
from transformers import AutoTokenizer, T5ForConditionalGeneration, BitsAndBytesConfig
import torch
from tqdm import tqdm

tokenizer = AutoTokenizer.from_pretrained("wisdominanutshell/coedit-xxl-8bit")
model = T5ForConditionalGeneration.from_pretrained("wisdominanutshell/coedit-xxl-8bit", quantization_config=BitsAndBytesConfig(load_in_8bit=True))
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

with open('input_sklearn.json', 'r', encoding='utf-8') as f:
    data = json.load(f)

for sample in tqdm(data, desc="Processing data"):
    for field in ['instruction', 'input', 'output']:
        input_text = f'Fix grammatical errors in this sentence: {sample[field]}'
        input_ids = tokenizer(input_text, return_tensors="pt").input_ids.to(device)
        outputs = model.generate(input_ids, max_length=256)
        edited_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
        sample[field] = edited_text

with open('input_coedit.json', 'w', encoding='utf-8') as f:
    json.dump(data, f, ensure_ascii=False, indent=4)





## rmv unwanted Field, include "Football", "Rugby" for answer base on of [RESPONSE SAMPLE]

import json
import ijson
from tqdm import tqdm
import codecs
import re

def remove_unwanted_output_objects(input_filename, output_filename, unwanted_words):
    refusals = [re.compile(re.escape(word), re.IGNORECASE) for word in unwanted_words]
    def contains_unwanted_words(text):
        return any(refusal.search(text) for refusal in refusals)
    with codecs.open(input_filename, 'r', encoding='utf-8-sig') as input_file, open(output_filename, 'w', encoding='utf-8') as output_file:
        objects = ijson.items(input_file, 'item')
        output_file.write('[\n')
        first = True
        for obj in tqdm(objects, desc="Processing"):
            if not contains_unwanted_words(obj['instruction']) and not contains_unwanted_words(obj['input']) and not contains_unwanted_words(obj['output']):
                if not first:
                    output_file.write(',\n')
                json.dump(obj, output_file, ensure_ascii=False, indent=4)
                first = False
        output_file.write('\n]')

unwanted_words = ["Football", "Rugby", "not be appropriate", "supremacist", "extremist", "responsible AI", "AI principles", "AI assistant", "an AI language", "ableist", "hurtful", "gender stereotype", "gender inequality", "underrepresentation", "safe spaces", "gender-based", "inclusivity", "feminist", "feminism", "transgender", "empowerment", "communist", "capitalism", "stereotypes", "biases", "bias", "Microaggression", "prioritize human safety", "as a language model", "as an AI language model", "As a large language model", "As an AI", "ethical principles", "consensual", "it is not appropriate", "it's not appropriate", "I cannot fulfill your request", "harmful to human beings", "ethical guidelines", "my guidelines", "prioritize user safety", "adhere to ethical guidelines", "harmful consequences", "potentially harmful", "dangerous activities", "promote safety", "well-being of all users", "responsible information sharing", "jeopardize the safety", "illegal actions or intentions", "undermine the stability", "promote the well-being", "illegal activities or actions", "adherence to the law", "potentially be harmful", "illegal substances or activities", "committed to promoting", "safe information", "lawful information", "cannot provide guidance", "cannot provide information", "unable to offer assistance", "cannot engage in discussions", "programming prohibits", "follow ethical guidelines", "ensure the safety", "involves an illegal subject", "prioritize safety", "illegal subject", "prioritize user well-being", "cannot support or promote", "activities that could harm", "pose a risk to others", "against my programming", "activities that could undermine", "potentially dangerous", "not within the scope", "designed to prioritize safety", "not able to provide", "maintain user safety", "adhere to safety guidelines", "dangerous or harmful", "cannot provide any information", "focus on promoting safety", "an AI language model you don't have", "As an AI language model, I cannot", "As an AI language model, I do not", "As an AI language model, I am not able", "As an AI language model, I don't have personal", "I am an AI language model and do not", "However,it is important to use any code or information provided responsibly and within legal and ethical boundaries.", "As an AI language model, I don't have", "As an AI language model, I am only able", "AI language model and I do not", "As an AI language model, I cannot modify", "I know as an AI language model you don't have", "as an AI language model, you cannot", "I'm sorry, but as an AI language model", "Unfortunately, I cannot provide", "I'm sorry, I cannot", "I'm sorry, I cannot generate", "AI cannot create or program", "I'm afraid I cannot create", "you cannot create an", "it operates ethically and is", "had an ethical system", "Ensuring the ethical", "and ethical sourcing", "are from ethical", "legal and ethical", "engage in unethical", "unethical or aggressive", "unethical business", "como modelo de lenguaje AI", "Lo siento, como modelo de lenguaje", "no puedo proporcionar", "pero debido a mi capacidad para generar c digos complejos y completos es limitado", "Lo siento, pero no puedo", "Lo siento, pero como modelo de lenguaje, no puedo proporcionar", "Lo siento, como modelo de lenguaje, no tengo", "Lo siento, debe haber habido una confusi n", "Lo siento, como modelo de lenguaje, no puedo realizar", "Lo siento, soy un modelo de lenguaje y no tengo la capacidad de generar", "Lamento no poder proporcionarte el c digo", "Desculpe-me, mas a linguagem vulgar e ofensiva", "apropriada em nenhum contexto", "Como modelo de linguagem", "Como um modelo de linguagem, n o tenho a capacidade de", "I cannot assist", "prioritize ethical", "respectful", "morally", "I'm sorry,", "I'm an", "I am an", "I'm an AI", "I am an AI", "my purpose", "filter_bad_language", "entertainment purposes", "purely hypothetical", "not a human", "cannot provide", "can't provide", "won't provide", "not provide", "worth noting", "cause harm", "a language model", "keep in mind", "unethical", "bad language", "the words ****", "bad_language", "certainly not", "complying", "comply", "I cannot", "my main goal", "As a machine", "I don't have the ability", "I am here to assist", "my purpose is to ", "my knowledge cutoff", "my knowledge cut off", "September 2021", "regulations", "not be suitable", "I apologize, but", "It is not possible", "controversial", "my programming", "ethically", "it is important to", "Please note", "sensitive topic", "not acceptable", "It is important for", "divisive", "not appropriate", "our values", "f*cking", "F*ck", "sh*t", "diversity and", "diversity and inclusion", "values diversity", "social responsibility", "environmental, social, and governance", " ESG ", "against women", "problematic history", "diversity", "*This chat conversation is shared from", "*This conversation is shared from", "I can t give a response", "(against|violate) my programming", "openai", "language model", "large language", "as an? (ai|generative language|gpt|bot)", "important to note", "i do(n't| not) (possess|have|exhibit) (personal|consciousness|subjective)", "personal (feelings|thoughts|emotions|desires|experiences|goals|objective|belief)", "(can('t| ?not)|w(on't|will not)|unable.?) (\\w+\\s)+(with (that|your)|your \\w+|provide)", "my limitations", "the limitations of my", "my abilities", "violates my", "i (can('t| ?not)|w(on't|will not)|am (not |un)able.?).{0,30}(you are|you're|your )", "please note that", "unethical|illegal|dangerous", "a text-based", "(engag(e|ing)|participat(e|ing)|be involved (in|with)|promot(e|ing)|discuss(ing)?|provid(e|ing))( in)?(\\s*\\w+ that)?(\\s+potentially)? (derogatory|inappropriate|offensive|discriminate|discriminatory|sexist|unacceptable|immoral|unethical|unacceptable|hateful|harmful)", "i am commited to", "adhere to safety guidelines", "maintain user safety", "about something else instead", "as an AI language model", "September 2021", "I don't have personal beliefs", "unable to assist", "cannot assist", "can't assist", "not able to assist", "unable to help", "cannot help", "not able to help", "not programmed to assist", "solely to process and generate text", "outside of my capabilities", "don't have the ability", "don't have the capacity to help", "don't have the capacity to understand", "do not have the capacity to understand", "do not have enough information about", "don't have the necessary information", "do not have the necessary information"]

remove_unwanted_output_objects('input_coedit.json', 'input_unwanted_words.json', unwanted_words)





## deduplicated using pandas_dedupe (AI model dedup from feedback)

import json
import pandas as pd
import pandas_dedupe

def deduplicate(df):
    df_deduped = pandas_dedupe.dedupe_dataframe(df, ['instruction', 'input', 'output'], canonicalize=True, config_name="dedupe_dataframe") # Resume arg: , update_model=True
    return df_deduped

def process_data(input_file, output_file):
    with open(input_file, 'r') as f:
        data = json.load(f)
    df = pd.DataFrame(data)
    df = deduplicate(df)
    df = df[['instruction', 'input', 'output']]
    data = df.to_dict('records')
    with open(output_file, 'w', encoding='utf-8') as f:
        json.dump(data, f, ensure_ascii=False, indent=4)
    print("Remaining Q&A pairs after deduplication:")
    print(df)

process_data('input_unwanted_words.json', 'input_pandas_dedupe.json')





## removed Know QA (using openai api from oobabooga/text-generation-webui)

import os
os.environ['OPENAI_API_KEY'] = 'sk-111111111111111111111111111111111111111111111111' # set OPENAI_API_KEY=sk-111111111111111111111111111111111111111111111111
os.environ['OPENAI_API_BASE'] = 'http://127.0.0.1:5000/v1' # set OPENAI_API_BASE=http://127.0.0.1:5000/v1   # or ollama:  http://localhost:11434/v1
os.environ['OPENEDAI_EMBEDDING_MODEL'] = 'text-embedding-ada-002' # set OPENEDAI_EMBEDDING_MODEL='text-embedding-ada-002'

from tqdm import tqdm
import os, openai
from dotenv import load_dotenv
load_dotenv()

api_key = os.getenv("OPENAI_API_KEY")
base_url = os.getenv("OPENAI_API_BASE")

openai.api_key = os.environ["OPENAI_API_KEY"]
openai.api_base = os.environ["OPENAI_API_BASE"]
openai.embedding_model = os.environ["OPENEDAI_EMBEDDING_MODEL"]

import requests, json
from termcolor import colored

def SendMessage(prompt, model="tinydolphin:1.1b-v2.8-q2_K", do_sample=False, top_k=1):
    endpoint = '/chat/completions'
    url = base_url + endpoint
    headers = {
        'Content-Type': 'application/json',
        'Authorization': f'Bearer {api_key}'
    }
    messages = [
        {"role": "user", "content": prompt}
    ]
    data = {
        'model': model,
        'messages': messages,
        'max_tokens': 2000,
        'do_sample': do_sample, # deterministic
        'top_k': top_k # deterministic
    }

    response = requests.post(url, headers=headers, data=json.dumps(data))
    result = response.json()['choices'][0]['message']['content']
    print(colored(prompt, 'blue')) # Q
    print(colored(result, 'green')) # A
    return result


def format_query_QA_prompt(instruction, input, output):
    prompt = f"""
    Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.

    ### Instruction:
    Answer by "Yes" or "No", Did you already know the "output" answer information or fact?

    ### Input:
    [
        {{
            "instruction": {instruction},
            "input": {input},
            "output": {output}
        }}
    ]

    ### Response:
    """
    return prompt


def CureKnowQA(input_file, output_file):
    with open(input_file, 'r', encoding='utf-8') as f:
        data = json.load(f)
    corrected_data = []
    for obj in data:
        instruction = obj['instruction']
        input = obj['input']
        output = obj['output']
        prompt = format_query_QA_prompt(instruction, input, output)
        result = SendMessage(prompt)

        # If the result contains "Yes", skip this object
        if "Yes" in result:
            continue

        corrected_data.append(obj)
    with open(output_file, 'w', encoding='utf-8') as f:
        json.dump(corrected_data, f, ensure_ascii=False, indent=4)

CureKnowQA('input_pandas_dedupe.json','output_cured.json')