Content Moderation System

import chromadb
from chromadb.config import Settings

# Initialize Chroma client
client = chromadb.Client(Settings(
    chroma_db_impl="duckdb+parquet",
    persist_directory="/path/to/persist"
))

# Create a collection for problematic content
collection = client.create_collection(name="problematic_content")

# Add some example problematic content
collection.add(
    documents=["hate speech example", "violent content example", "spam content example"],
    metadatas=[{"category": "hate"}, {"category": "violence"}, {"category": "spam"}],
    ids=["1", "2", "3"]
)

# Function to check new content
def check_content(new_post):
    results = collection.query(
        query_texts=[new_post],
        n_results=1
    )

    if results['distances'][0][0] < 0.5:  # Adjust threshold as needed
        return f"Flagged as potentially {results['metadatas'][0][0]['category']}"
    else:
        return "Content seems appropriate"

# Test the system
print(check_content("This is a normal post"))
print(check_content("This post contains hate speech"))