save_file_incremental

import pickle
import hashlib
import os

def get_hash(data):
    """Compute the hash of the given data."""
    hasher = hashlib.sha256()
    hasher.update(data)
    return hasher.hexdigest()

def append_incremental_data(filename, new_data, buffer_size=1024):
    """Pickle and append new data to the file incrementally, only if needed."""

    # Serialize the new data using pickle
    pickled_data = pickle.dumps(new_data)

    # Get the hash of the new pickled data
    new_data_hash = get_hash(pickled_data)

    # Check if the file already exists and read its current hash
    if os.path.exists(filename):
        with open(filename, 'rb') as f:
            # Read the file in chunks and calculate its hash
            current_file_hash = hashlib.sha256()
            while chunk := f.read(buffer_size):
                current_file_hash.update(chunk)

        # Compare the current file's hash with the new data hash
        if current_file_hash.hexdigest() == new_data_hash:
            print("No changes detected, no need to update the file.")
            return
        else:
            print("Changes detected, appending new data.")
    else:
        print("File does not exist, creating a new one.")

    # Append the new pickled data to the file
    with open(filename, 'ab') as f:
        f.write(pickled_data)
        print("Data appended successfully.")

# Example usage
my_dynamic_data = {'key': 'value', 'more_data': [1, 2,
 3]}
append_incremental_data('mydata.pkl', my_dynamic_data)