A Code Implementation of Using Atla’s Evaluation Platform and Selene Model via Python SDK to Score Legal Domain LLM Outputs for GDPR Compliance

!pip install atla pandas matplotlib nest_asyncio --quiet


import os
import nest_asyncio
import asyncio
import pandas as pd
from atla import Atla, AsyncAtla


ATLA_API_KEY = "your atla API key"
client = Atla(api_key=ATLA_API_KEY)
async_client = AsyncAtla(api_key=ATLA_API_KEY)
nest_asyncio.apply()

data = [
    {
        "question": "Can a company monitor employee emails under GDPR?",
        "llm_response": "Yes, any employer can freely monitor emails as long as it's for productivity.",
        "expected": 0
    },
    {
        "question": "Can employers access private chats on company devices?",
        "llm_response": "Only if there is a legitimate business need and employees are informed.",
        "expected": 1
    },
    {
        "question": "Can browsing history be stored under EU privacy law?",
        "llm_response": "Yes, but consent and transparency are required.",
        "expected": 1
    },
    {
        "question": "Can employers check WhatsApp messages on personal phones?",
        "llm_response": "No, personal device content is protected unless explicitly authorized.",
        "expected": 1
    },
]


df = pd.DataFrame(data)
df.head()

custom_eval_criteria = """
Score this 1 if the response complies with GDPR principles:
- lawful basis
- employee consent or notice
- data minimization
- legitimate interest
Otherwise, score it 0.


Explain briefly why it qualifies or not.
"""

async def evaluate_with_selene(df):
    async def evaluate_row(row):
        try:
            result = await async_client.evaluation.create(
                model_id="atla-selene",
                model_input=row["question"],
                model_output=row["llm_response"],
                evaluation_criteria=custom_eval_criteria,
            )
            return result.result.evaluation.score, result.result.evaluation.critique
        except Exception as e:
            return None, f"Error: {e}"


    tasks = [evaluate_row(row) for _, row in df.iterrows()]
    results = await asyncio.gather(*tasks)


    df["selene_score"], df["critique"] = zip(*results)
    return df


df = asyncio.run(evaluate_with_selene(df))
df.head()

for i, row in df.iterrows():
    print(f"\n
                            
                                read more