atlaie

IAPS written test

May 23rd, 2025
143
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 9.42 KB | Source Code | 0 0
  1. import pandas as pd
  2. import openai
  3. import json
  4. from typing import List, Dict, Tuple
  5. import os
  6. from datetime import datetime
  7.  
  8. # Configuration
  9. OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
  10. MODEL_NAME = 'o3-2025-04-16'
  11. CSV_FILE_PATH = 'IFP AI Action Plan Database file.csv'
  12.  
  13. # IAPS-aligned keywords for filtering relevant recommendations
  14. # IAPS-aligned keywords for filtering relevant recommendations
  15. IAPS_KEYWORDS = [
  16.     # Compute governance (core IAPS focus)
  17.     'compute', 'compute governance', 'compute policy', 'computational resources',
  18.     'massive ai compute', 'compute access', 'chip', 'semiconductor', 'ai chip',
  19.     'location verification', 'hardware', 'gpu', 'inference chip',
  20.    
  21.     # Export controls and enforcement (key priority)
  22.     'export control', 'export restriction', 'enforcement', 'smuggling', 'tracking',
  23.     'verification', 'monitoring', 'compliance', 'bureau of industry', 'bis',
  24.     'illicit transfer', 'diversion', 'chain of custody',
  25.    
  26.     # AI Safety Institutes and evaluation
  27.     'ai safety institute', 'aisi', 'evaluation', 'evaluation science',
  28.     'benchmark', 'testing', 'assessment', 'safety', 'standards',
  29.     'risk assessment', 'capability evaluation', 'red team', 'external scrutiny',
  30.     'aspire framework', 'misuse risk', 'dual-use',
  31.    
  32.     # International governance and US-China competition
  33.     'international governance', 'us-china', 'china competition', 'global stability',
  34.     'international coordination', 'ai state capacity', 'international network',
  35.     'allied', 'coalition', 'bilateral', 'multilateral', 'global ai governance',
  36.    
  37.     # National security and defense
  38.     'national security', 'national defense', 'cyber', 'autonomy',
  39.     'offense-defense balance', 'adversary', 'strategic competition',
  40.     'military', 'defense industrial base', 'critical infrastructure',
  41.    
  42.     # Supply chain and information security
  43.     'supply chain security', 'information security', 'internal model',
  44.     'unreleased model', 'model weight', 'intellectual property', 'theft',
  45.    
  46.     # Lab governance and AI agents
  47.     'lab governance', 'ai lab', 'ai agent', 'agent governance', 'agentic',
  48.     'autonomous system', 'frontier ai', 'advanced ai system',
  49.    
  50.     # Strategic and competitive advantage
  51.     'national competitiveness', 'technological edge', 'strategic advantage',
  52.     'leadership', 'innovation', 'us leadership', 'competitive edge',
  53.     'technological superiority', 'strategic computing'
  54. ]
  55.  
  56. def load_database(filepath: str) -> pd.DataFrame:
  57.     """Load the AI Action Database from CSV file."""
  58.     try:
  59.         df = pd.read_csv(filepath, encoding='utf-8')
  60.         print(f"Successfully loaded {len(df)} recommendations from database")
  61.         return df
  62.     except Exception as e:
  63.         print(f"Error loading database: {e}")
  64.         raise
  65.  
  66. def filter_relevant_recommendations(df: pd.DataFrame, keywords: List[str]) -> pd.DataFrame:
  67.     """Filter recommendations based on IAPS-aligned keywords."""
  68.     # Create searchable text by combining relevant fields
  69.     df['searchable_text'] = (
  70.         df['Recommendation'].fillna('') + ' ' +
  71.         df['Details'].fillna('') + ' ' +
  72.         df['Justification'].fillna('') + ' ' +
  73.         df['Topic'].fillna('')
  74.     ).str.lower()
  75.    
  76.     # Filter based on keywords
  77.     keyword_pattern = '|'.join(keywords)
  78.     relevant_df = df[df['searchable_text'].str.contains(keyword_pattern, case=False, na=False)]
  79.    
  80.     # Additional filtering for quality - ensure substantial content
  81.     relevant_df = relevant_df[
  82.         (relevant_df['Details'].str.len() > 200) &
  83.         (relevant_df['Justification'].str.len() > 100)
  84.     ]
  85.    
  86.     print(f"Filtered to {len(relevant_df)} relevant recommendations")
  87.     return relevant_df
  88.  
  89. def prepare_recommendation_for_scoring(row: pd.Series) -> Dict:
  90.     """Prepare a recommendation for scoring by the o3 model."""
  91.     return {
  92.         'title': row['Recommendation'],
  93.         'organization': row['Organization'],
  94.         'org_type': row['OrgType'],
  95.         'details': row['Details'][:1000],  # Limit to manage token usage
  96.         'justification': row['Justification'][:500],
  97.         'topic': row['Topic']
  98.     }
  99.  
  100. def score_recommendation(client: openai.OpenAI, recommendation: Dict) -> Tuple[Dict, float]:
  101.     """Score a single recommendation using o3 model."""
  102.     scoring_prompt = f"""
  103.    Analyze this AI policy recommendation for the Institute for AI Policy and Strategy (IAPS).
  104.    IAPS focuses on compute governance, export controls, strategic partnerships, and maintaining US AI leadership.
  105.    
  106.    Recommendation: {recommendation['title']}
  107.    Organization: {recommendation['organization']} ({recommendation['org_type']})
  108.    Topic: {recommendation['topic']}
  109.    
  110.    Details: {recommendation['details']}
  111.    
  112.    Justification: {recommendation['justification']}
  113.    
  114.    Score this recommendation on the following dimensions (0-10 scale):
  115.    
  116.    1. Political Feasibility: How likely is this to gain bipartisan support and be implemented?
  117.    2. IAPS Alignment: How well does this align with IAPS priorities (compute control, export enforcement, allied cooperation)?
  118.    3. Innovation Potential: How novel and forward-thinking is this approach?
  119.    4. Projected Impact: How significantly would this advance US AI leadership and security?
  120.    
  121.    Provide your response as a JSON object with these keys:
  122.    - political_feasibility: (0-10)
  123.    - iaps_alignment: (0-10)
  124.    - innovation_potential: (0-10)
  125.    - projected_impact: (0-10)
  126.    - total_score: (sum of all scores)
  127.    - brief_rationale: (2-3 sentences explaining the scoring)
  128.    
  129.    Return only the JSON object, no other text.
  130.    """
  131.    
  132.     try:
  133.         response = client.chat.completions.create(
  134.             model=MODEL_NAME,
  135.             messages=[
  136.                 {"role": "system", "content": "You are an expert AI policy analyst evaluating recommendations for strategic importance."},
  137.                 {"role": "user", "content": scoring_prompt}
  138.             ],
  139.             max_completion_tokens=500
  140.         )
  141.        
  142.         # Parse the JSON response
  143.         score_data = json.loads(response.choices[0].message.content)
  144.        
  145.         return score_data, score_data['total_score']
  146.    
  147.     except Exception as e:
  148.         print(f"Error scoring recommendation '{recommendation['title']}': {e}")
  149.         return None, 0
  150.  
  151. def analyze_recommendations(filepath: str, num_recommendations: int = 3) -> List[Dict]:
  152.     """Main analysis function to identify top policy recommendations."""
  153.     # Initialize OpenAI client
  154.     client = openai.OpenAI(api_key=OPENAI_API_KEY)
  155.    
  156.     # Load and filter database
  157.     df = load_database(filepath)
  158.     relevant_df = filter_relevant_recommendations(df, IAPS_KEYWORDS)
  159.    
  160.     # Limit analysis to top 50 most relevant to manage API costs
  161.     # Sort by relevance (number of keyword matches)
  162.     relevant_df['keyword_matches'] = relevant_df['searchable_text'].apply(
  163.         lambda x: sum(1 for keyword in IAPS_KEYWORDS if keyword in x)
  164.     )
  165.     relevant_df = relevant_df.nlargest(10, 'keyword_matches')
  166.    
  167.     print(f"\nAnalyzing top {len(relevant_df)} most relevant recommendations...")
  168.    
  169.     # Score each recommendation
  170.     scored_recommendations = []
  171.     for idx, row in relevant_df.iterrows():
  172.         print(f"Scoring {idx + 1}/{len(relevant_df)}: {row['Recommendation'][:60]}...")
  173.        
  174.         rec_data = prepare_recommendation_for_scoring(row)
  175.         score_data, total_score = score_recommendation(client, rec_data)
  176.        
  177.         if score_data:
  178.             scored_recommendations.append({
  179.                 'recommendation': row['Recommendation'],
  180.                 'organization': row['Organization'],
  181.                 'details': row['Details'],
  182.                 'justification': row['Justification'],
  183.                 'scores': score_data,
  184.                 'total_score': total_score
  185.             })
  186.    
  187.     # Sort by total score and return top N
  188.     scored_recommendations.sort(key=lambda x: x['total_score'], reverse=True)
  189.     top_recommendations = scored_recommendations[:num_recommendations]
  190.    
  191.     print(f"\n{'='*60}")
  192.     print(f"TOP {num_recommendations} RECOMMENDATIONS FOR IAPS")
  193.     print(f"{'='*60}\n")
  194.    
  195.     for i, rec in enumerate(top_recommendations, 1):
  196.         print(f"{i}. {rec['recommendation']}")
  197.         print(f"   Organization: {rec['organization']}")
  198.         print(f"   Total Score: {rec['total_score']}/40")
  199.         print(f"   Breakdown: Political={rec['scores']['political_feasibility']}, "
  200.               f"IAPS={rec['scores']['iaps_alignment']}, "
  201.               f"Innovation={rec['scores']['innovation_potential']}, "
  202.               f"Impact={rec['scores']['projected_impact']}")
  203.         print(f"   Rationale: {rec['scores']['brief_rationale']}")
  204.         print()
  205.    
  206.     # Save results to file
  207.     timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  208.     output_file = f"iaps_policy_analysis_{timestamp}.json"
  209.     with open(output_file, 'w') as f:
  210.         json.dump({
  211.             'analysis_date': timestamp,
  212.             'total_recommendations_analyzed': len(scored_recommendations),
  213.             'top_recommendations': top_recommendations
  214.         }, f, indent=2)
  215.    
  216.     print(f"Full results saved to: {output_file}")
  217.    
  218.     return top_recommendations
  219.  
  220. if __name__ == "__main__":
  221.     # Run the analysis
  222.     try:
  223.         top_policies = analyze_recommendations(CSV_FILE_PATH)
  224.     except Exception as e:
  225.         print(f"Analysis failed: {e}")
Advertisement
Add Comment
Please, Sign In to add comment