Verify_Alpaca_format.py

import ijson
def verify_json_file(filename):
    try:
        with open(filename, 'r', encoding='utf-8') as file: # Parse the JSON objects from the file one by one
            objects = ijson.items(file, 'item')
            for obj in objects:
                pass  # No need to do anything with the object, we just want to check if it loads
        print("All objects in the file were successfully loaded in UTF-8 as JSON format.")
    except ijson.JSONError:
        print("An error occurred while loading the JSON data. The file may not be properly formatted.")
    except UnicodeDecodeError:
        print("An error occurred while decoding the file. The file may not be in UTF-8 encoding.")

verify_json_file('Base_zh_Alpaca-CoT.json')


import json
def verify_json(file_path):
    with open(file_path, 'r', encoding='utf-8-sig') as f:
        data = json.load(f)

    for i, obj in enumerate(data):
        assert 'instruction' in obj, f"Object at index {i} is missing 'instruction'"
        assert 'input' in obj, f"Object at index {i} is missing 'input'"
        assert 'output' in obj, f"Object at index {i} is missing 'output'"
    print("The JSON file is valid.")

# verify_json('Base_zh_Alpaca-CoT.json')