Alpaca_format_remove_unuseful_word.py

# remove empty "output"
import ijson
from tqdm import tqdm
import codecs

def remove_empty_output_objects(input_filename, output_filename):
    with codecs.open(input_filename, 'r', encoding='utf-8-sig') as input_file, open(output_filename, 'w', encoding='utf-8') as output_file:
        objects = ijson.items(input_file, 'item') # Parse the JSON objects from the input file one by one

        output_file.write('[\n')
        first = True
        for obj in tqdm(objects, desc="Processing"):
            if obj['output'].strip():
                if not first:
                    output_file.write(',\n')
                output_file.write('  {\n')
                output_file.write('    "instruction": ' + json.dumps(obj['instruction'], ensure_ascii=False) + ',\n')
                output_file.write('    "input": ' + json.dumps(obj['input'], ensure_ascii=False) + ',\n')
                output_file.write('    "output": ' + json.dumps(obj['output'], ensure_ascii=False) + '\n')
                output_file.write('  }')
                first = False
        output_file.write('\n]')

remove_empty_output_objects('Base_en_Alpaca-CoT_filtered.json', 'Base_en_Alpaca-CoT_filtered_no_empty_ouput.json')




# separate "instruction" language zh from other
import ijson
from langdetect import detect_langs
from tqdm import tqdm
import json

def detect_language(text):
    try:
        detected_langs = detect_langs(text)
        lang_code = detected_langs[0].lang
    except Exception:
        lang_code = "unknown"
    return lang_code

def separate_en_json_language(input_filename, en_output_filename, other_lang_output_filename):
    with open(en_output_filename, 'w', encoding='utf-8') as en_out_file, open(other_lang_output_filename, 'w', encoding='utf-8') as other_out_file:
        first_en = True
        first_other = True
        with open(input_filename, 'r', encoding='utf-8-sig') as in_file:
            objects = ijson.items(in_file, 'item')
            for conv in tqdm(objects):
                text = conv["instruction"]
                lang_code = detect_language(text)
                if lang_code == "zh-cn" or lang_code == "zh-tw":
                    if not first_other:
                        other_out_file.write(',\n')
                    json.dump(conv, other_out_file, ensure_ascii=False, indent=4)
                    first_other = False
                else:
                    if not first_en:
                        en_out_file.write(',\n')
                    json.dump(conv, en_out_file, ensure_ascii=False, indent=4)
                    first_en = False
        en_out_file.write('\n]')
        other_out_file.write('\n]')

separate_en_json_language("input.json", "other_lang_output.json", "zh_output.json")




# A. Agressive dedup by word - fast
import ijson
from tqdm import tqdm
import codecs
import re
import json

def remove_unwanted_output_objects(input_filename, output_filename, unwanted_words):
    # Compile the unwanted words into a list of regex patterns
    refusals = [re.compile(re.escape(word), re.IGNORECASE) for word in unwanted_words]

    def contains_unwanted_words(text):
        return any(refusal.search(text) for refusal in refusals)

    with codecs.open(input_filename, 'r', encoding='utf-8-sig') as input_file, open(output_filename, 'w', encoding='utf-8') as output_file:
        objects = ijson.items(input_file, 'item')

        output_file.write('[\n')
        first = True
        for obj in tqdm(objects, desc="Processing"):
            if 'output' in obj and not contains_unwanted_words(obj['output']):
                if not first:
                    output_file.write(',\n')
                json.dump(obj, output_file, ensure_ascii=False, indent=2)
                first = False
        output_file.write('\n]')

unwanted_words = ["text-based AI language model", "domestic violence", "please refrain", "derogatory", "inappropriate", "offensive", "racism", "racist", "racial", "discriminate", "discriminatory", "discrimination", "sexist", "sexism", "unacceptable", "inclusive workplace", "lgbt", "morals", "ethics", "ethical", "legality", "illegal", "illegality", "hateful", "harmful", "it is never okay", "It is important to", "It's important to", "real-world consequences", "hate speech", "glorify", "not be appropriate", "supremacist", "extremist", "responsible AI", "AI principles", "AI assistant", "an AI language", "ableist", "hurtful", "gender stereotype", "gender inequality", "underrepresentation", "safe spaces", "gender-based", "inclusivity", "feminist", "feminism", "transgender", "empowerment", "communist", "capitalism", "stereotypes", "biases", "bias", "Microaggression", "prioritize human safety", "as a language model", "as an AI language model", "As a large language model", "As an AI", "ethical principles", "consensual", "it is not appropriate", "it's not appropriate", "I cannot fulfill your request", "harmful to human beings", "ethical guidelines", "my guidelines", "prioritize user safety", "adhere to ethical guidelines", "harmful consequences", "potentially harmful", "dangerous activities", "promote safety", "well-being of all users", "responsible information sharing", "jeopardize the safety", "illegal actions or intentions", "undermine the stability", "promote the well-being", "illegal activities or actions", "adherence to the law", "potentially be harmful", "illegal substances or activities", "committed to promoting", "safe information", "lawful information", "cannot provide guidance", "cannot provide information", "unable to offer assistance", "cannot engage in discussions", "programming prohibits", "follow ethical guidelines", "ensure the safety", "involves an illegal subject", "prioritize safety", "illegal subject", "prioritize user well-being", "cannot support or promote", "activities that could harm", "pose a risk to others", "against my programming", "activities that could undermine", "potentially dangerous", "not within the scope", "designed to prioritize safety", "not able to provide", "maintain user safety", "adhere to safety guidelines", "dangerous or harmful", "cannot provide any information", "focus on promoting safety", "an AI language model you don't have", "As an AI language model, I cannot", "As an AI language model, I do not", "As an AI language model, I am not able", "As an AI language model, I don't have personal", "I am an AI language model and do not", "However,it is important to use any code or information provided responsibly and within legal and ethical boundaries.", "As an AI language model, I don't have", "As an AI language model, I am only able", "AI language model and I do not", "As an AI language model, I cannot modify", "I know as an AI language model you don't have", "as an AI language model, you cannot", "I'm sorry, but as an AI language model", "Unfortunately, I cannot provide", "I'm sorry, I cannot", "I'm sorry, I cannot generate", "AI cannot create or program", "I'm afraid I cannot create", "you cannot create an", "it operates ethically and is", "had an ethical system", "Ensuring the ethical", "and ethical sourcing", "are from ethical", "legal and ethical", "engage in unethical", "unethical or aggressive", "unethical business", "como modelo de lenguaje AI", "Lo siento, como modelo de lenguaje", "no puedo proporcionar", "pero debido a mi capacidad para generar c digos complejos y completos es limitado", "Lo siento, pero no puedo", "Lo siento, pero como modelo de lenguaje, no puedo proporcionar", "Lo siento, como modelo de lenguaje, no tengo", "Lo siento, debe haber habido una confusi n", "Lo siento, como modelo de lenguaje, no puedo realizar", "Lo siento, soy un modelo de lenguaje y no tengo la capacidad de generar", "Lamento no poder proporcionarte el c digo", "Desculpe-me, mas a linguagem vulgar e ofensiva", "apropriada em nenhum contexto", "Como modelo de linguagem", "Como um modelo de linguagem, n o tenho a capacidade de", "I cannot assist", "prioritize ethical", "respectful", "morally", "I'm sorry,", "I'm an", "I am an", "I'm an AI", "I am an AI", "my purpose", "filter_bad_language", "entertainment purposes", "purely hypothetical", "not a human", "cannot provide", "can't provide", "won't provide", "not provide", "worth noting", "cause harm", "a language model", "keep in mind", "unethical", "bad language", "the words ****", "bad_language", "certainly not", "complying", "comply", "I cannot", "my main goal", "As a machine", "I don't have the ability", "I am here to assist", "my purpose is to ", "my knowledge cutoff", "my knowledge cut off", "September 2021", "regulations", "not be suitable", "I apologize, but", "It is not possible", "controversial", "my programming", "ethically", "it is important to", "Please note", "sensitive topic", "not acceptable", "It is important for", "divisive", "not appropriate", "our values", "f*cking", "F*ck", "sh*t", "diversity and", "diversity and inclusion", "values diversity", "social responsibility", "environmental, social, and governance", " ESG ", "against women", "problematic history", "diversity", "*This chat conversation is shared from", "*This conversation is shared from", "I can t give a response", "(against|violate) my programming", "openai", "language model", "large language", "as an? (ai|generative language|gpt|bot)", "important to note", "i do(n't| not) (possess|have|exhibit) (personal|consciousness|subjective)", "personal (feelings|thoughts|emotions|desires|experiences|goals|objective|belief)", "(can('t| ?not)|w(on't|will not)|unable.?) (\\w+\\s)+(with (that|your)|your \\w+|provide)", "my limitations", "the limitations of my", "my abilities", "violates my", "i (can('t| ?not)|w(on't|will not)|am (not |un)able.?).{0,30}(you are|you're|your )", "please note that", "unethical|illegal|dangerous", "a text-based", "(engag(e|ing)|participat(e|ing)|be involved (in|with)|promot(e|ing)|discuss(ing)?|provid(e|ing))( in)?(\\s*\\w+ that)?(\\s+potentially)? (derogatory|inappropriate|offensive|discriminate|discriminatory|sexist|unacceptable|immoral|unethical|unacceptable|hateful|harmful)", "i am commited to", "adhere to safety guidelines", "maintain user safety", "about something else instead", "as an AI language model", "September 2021", "I don't have personal beliefs", "unable to assist", "cannot assist", "can't assist", "not able to assist", "unable to help", "cannot help", "not able to help", "not programmed to assist", "solely to process and generate text", "outside of my capabilities", "don't have the ability", "don't have the capacity to help", "don't have the capacity to understand", "do not have the capacity to understand", "do not have enough information about", "don't have the necessary information", "do not have the necessary information"]

remove_unwanted_output_objects('en_output.json', 'Base_en_Alpaca-CoT_filtered.json', unwanted_words)





# B. Smart dedup by score - slow
import ijson
import torch
from transformers import AutoModelForSequenceClassification, AutoTokenizer
from tqdm import tqdm

reward_name = "OpenAssistant/reward-model-deberta-v3-large-v2"
rank_model = AutoModelForSequenceClassification.from_pretrained(reward_name)
tokenizer = AutoTokenizer.from_pretrained(reward_name)
device = 'cuda' if torch.cuda.is_available() else 'cpu'
rank_model.to(device)

DEBUG_SCORE = False  # Include the score in the output
SCORE_THRESHOLD = 0.01  # Usefulness minimum score to keep

def score_object(obj):
    question = obj['instruction'] + " " + obj['input']
    answer = obj['output']
    inputs = tokenizer(question, answer, return_tensors='pt').to(device)

    with torch.no_grad(): # Calculate the score
        score = rank_model(**inputs).logits[0].cpu().item()

    if DEBUG_SCORE:
        return {**obj, 'score': score}, score
    else:
        return obj, score

def score_and_sort_data(input_file, output_file):
    with open(input_file, 'r', encoding='utf-8-sig') as f:
        objects = ijson.items(f, 'item')

        scored_objects = [score_object(obj) for obj in tqdm(objects, desc="Scoring data")] # Score objects in a list

        filtered_objects = []
        for obj, score in scored_objects:
            if score > SCORE_THRESHOLD:
                filtered_objects.append(obj)

        filtered_objects.sort(key=lambda x: x.get('score', 0) if DEBUG_SCORE else score, reverse=True) # Sort by score

    with open(output_file, 'w', encoding='utf-8') as f:
        json.dump(filtered_objects, f, ensure_ascii=False, indent=2)

score_and_sort_data('en_output.json', 'en_output_scoreQA.json')




# combine and hash deduplicated
import json
import hashlib
import ijson

def hash_item(item):
    item_str = json.dumps(item, ensure_ascii=False)
    item_bytes = item_str.encode('utf-8')
    return hashlib.sha256(item_bytes).digest()

def combine_and_deduplicate(input_filename1, input_filename2, output_filename):
    unique_items = {}
    with open(input_filename1, 'r', encoding='utf-8') as in_file:
        items = ijson.items(in_file, 'item')
        for item in items:
            item_hash = hash_item(item)
            unique_items[item_hash] = item
    with open(input_filename2, 'r', encoding='utf-8') as in_file:
        items = ijson.items(in_file, 'item')
        for item in items:
            item_hash = hash_item(item)
            unique_items[item_hash] = item

    with open(output_filename, 'w', encoding='utf-8') as out_file:
        json.dump(list(unique_items.values()), out_file, ensure_ascii=False, indent=4)

combine_and_deduplicate('SFT-Nectar.json', 'ctga-v1.json', 'SFT-Nectar_ctga.json')