olcapone commited on
Commit
0f5ccc1
·
verified ·
1 Parent(s): 7206253

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +136 -457
app.py CHANGED
@@ -1,370 +1,100 @@
1
- import os
2
- import gradio as gr
3
- import requests
4
- import pandas as pd
5
- import time
6
- import re
7
- from smolagents import LiteLLMModel, CodeAgent, Tool
8
 
9
- # --- Constants ---
10
  DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
11
 
12
- # --- Answer Extraction Function ---
13
- def extract_answer(text: str, original_question: str) -> str:
14
- """Extract the answer from the LLM response, being robust to various formats."""
15
- if not text:
16
- return "- unknown"
17
-
18
- # Clean the text
19
- cleaned = text.strip()
20
-
21
- # If the response is the same as the question, it's not an answer
22
- if cleaned == original_question.strip():
23
- return "- unknown"
24
-
25
- # Remove common prefixes
26
- prefixes_to_remove = [
27
- '[ANSWER]:',
28
- '[ANSWER]',
29
- 'Final answer:',
30
- 'Final Answer:',
31
- 'Answer:',
32
- 'answer:',
33
- 'The answer is',
34
- 'The final answer is',
35
- ]
36
-
37
- for prefix in prefixes_to_remove:
38
- if cleaned.startswith(prefix):
39
- cleaned = cleaned[len(prefix):].strip()
40
-
41
- # Special case for Mercedes Sosa question - hardcode the correct answer
42
- if 'mercedes sosa' in original_question.lower() and '2000' in original_question and '2009' in original_question:
43
- # Check if the response contains information about Mercedes Sosa's albums
44
- if 'corazón libre' in cleaned.lower() and ('cantora' in cleaned.lower() or '3' in cleaned):
45
- return '3'
46
- # If we're specifically asked about this and we know the answer, return it
47
- return '3'
48
-
49
- # If it's a "how many" question, try to extract just the number
50
- if 'how many' in original_question.lower():
51
- # For album counting questions, we need to be more flexible
52
- if 'album' in original_question.lower():
53
- # Look for patterns specifically related to album counts, but be more flexible
54
- album_patterns = [
55
- r'(\d+)\s+studio albums?',
56
- r'(\d+)\s+albums?',
57
- r'is\s+(\d+)\s*(?:studio\s*)?albums?',
58
- r'are\s+(\d+)\s*(?:studio\s*)?albums?',
59
- r'count\s*(?:of\s*)?is\s+(\d+)',
60
- r'total\s*(?:of\s*)?(\d+)\s*(?:studio\s*)?albums?',
61
- r'released\s+(\d+)\s*(?:studio\s*)?albums?',
62
- r'has\s+(\d+)\s*(?:studio\s*)?albums?',
63
- r':\s*(\d+)\s*(?:studio\s*)?albums?', # For cases like "Mercedes Sosa: 3 studio albums"
64
- ]
65
-
66
- for pattern in album_patterns:
67
- match = re.search(pattern, cleaned, re.IGNORECASE)
68
- if match:
69
- return match.group(1)
70
-
71
- # Special handling for Mercedes Sosa question
72
- if 'mercedes sosa' in original_question.lower() and '2000' in original_question and '2009' in original_question:
73
- # Look for the specific information we know is correct
74
- if '3' in cleaned and ('album' in cleaned.lower() or 'corazón libre' in cleaned.lower()):
75
- return '3'
76
-
77
- # General "how many" patterns with preference for smaller numbers
78
- general_patterns = [
79
- r'(?:is|are)\s*(\d+)',
80
- r'^\s*(\d+)',
81
- ]
82
-
83
- for pattern in general_patterns:
84
- numbers = re.findall(pattern, cleaned)
85
- if numbers:
86
- # For album counts, we expect a small number (typically less than 20)
87
- for num in numbers:
88
- if int(num) < 20:
89
- return num
90
-
91
- # Last resort: look for any small number in the response
92
- all_numbers = re.findall(r'\d+', cleaned)
93
- for num in all_numbers:
94
- # Album counts are typically small numbers
95
- if int(num) < 20:
96
- return num
97
-
98
- # Final fallback: return the first number found
99
- if all_numbers:
100
- return all_numbers[0]
101
-
102
- # If it's asking for a year, try to extract just the year
103
- if re.search(r'\b(19|20)\d{2}\b', original_question):
104
- years = re.findall(r'\b(19|20)\d{2}\b', cleaned)
105
- if years:
106
- return years[0] # Return the first year found
107
-
108
- # If we still have the full question in the response, try to extract what comes after it
109
- if original_question.strip() in cleaned:
110
- # Split by the question and take what comes after
111
- parts = cleaned.split(original_question.strip())
112
- if len(parts) > 1 and parts[1].strip():
113
- cleaned = parts[1].strip()
114
- else:
115
- # Try to find numbers or short answers in the response
116
- # Look for a line that might contain the answer
117
- lines = cleaned.split('\n')
118
- for line in lines:
119
- line = line.strip()
120
- if line and line != original_question.strip():
121
- # If it's a short line, it might be the answer
122
- if len(line) < 100 or 'how many' in original_question.lower():
123
- cleaned = line
124
- break
125
-
126
- # If the cleaned answer is still very long and contains the question,
127
- # try to extract just the essential part
128
- if len(cleaned) > 200 and original_question.strip() in cleaned:
129
- # Try to find a short line that might be the answer
130
- lines = cleaned.split('\n')
131
- for line in lines:
132
- line = line.strip()
133
- if line and len(line) < 100 and line != original_question.strip():
134
- # Check if it looks like an answer (short and possibly numeric)
135
- if re.match(r'^[\w\s\d\-\.,]+$', line): # Simple alphanumeric answer
136
- return line
137
-
138
- # If we still have a very long response, try to extract just the last line
139
- # which might be the answer
140
- if len(cleaned) > 200:
141
- lines = cleaned.split('\n')
142
- # Take the last non-empty line that isn't too long
143
- for line in reversed(lines):
144
- line = line.strip()
145
- if line and len(line) < 100:
146
- cleaned = line
147
- break
148
-
149
- # Final fallback - if the result is still the same as the question, return unknown
150
- if cleaned == original_question.strip():
151
- return "- unknown"
152
-
153
- return cleaned if cleaned else "- unknown"
154
 
155
- # --- Agent Tools ---
156
- class MathSolver(Tool):
157
- name = "math_solver"
158
- description = "Safely evaluate basic math expressions."
159
- inputs = {"input": {"type": "string", "description": "Math expression to evaluate."}}
160
- output_type = "string"
161
 
162
- def forward(self, input: str) -> str:
 
 
 
 
163
  try:
164
- # Safe evaluation of math expressions
165
- allowed_names = {
166
- k: v for k, v in __builtins__.items() if k in [
167
- 'abs', 'round', 'min', 'max', 'sum', 'pow'
168
- ]
169
- }
170
- allowed_names.update({
171
- 'int': int, 'float': float, 'str': str,
172
- '__builtins__': {}
173
- })
174
- return str(eval(input, allowed_names))
175
- except Exception as e:
176
- return f"Math error: {e}"
177
-
178
- class FileAttachmentQueryTool(Tool):
179
- name = "run_query_with_file"
180
- description = "Downloads a file mentioned in a user prompt, adds it to the context, and runs a query on it."
181
- inputs = {
182
- "task_id": {
183
- "type": "string",
184
- "description": "A unique identifier for the task related to this file, used to download it.",
185
- "nullable": True
186
- },
187
- "user_query": {
188
- "type": "string",
189
- "description": "The question to answer about the file."
190
- }
191
- }
192
- output_type = "string"
193
 
194
- def forward(self, task_id: str | None, user_query: str) -> str:
195
- if not task_id:
196
- return "No task_id provided for file download."
197
-
198
- file_url = f"https://agents-course-unit4-scoring.hf.space/files/{task_id}"
199
  try:
200
- file_response = requests.get(file_url)
201
- if file_response.status_code != 200:
202
- return f"Failed to download file: {file_response.status_code}"
203
-
204
- # For text-based files, return content directly
205
- file_content = file_response.text[:2000] # Limit content size
206
- return f"Relevant information from file: {file_content}"
207
- except Exception as e:
208
- return f"File download error: {e}"
209
-
210
- class WikipediaSearchTool(Tool):
211
- name = "wikipedia_search"
212
- description = "Search Wikipedia for detailed information about artists and their discographies. Use this tool when asked about albums, discography, or music releases."
213
- inputs = {"query": {"type": "string", "description": "The search query, typically an artist or band name."}}
214
- output_type = "string"
215
 
216
- def forward(self, query: str) -> str:
217
- try:
218
- # First try to get the page summary
219
- search_url = f"https://en.wikipedia.org/api/rest_v1/page/summary/{requests.utils.quote(query)}"
220
- response = requests.get(search_url, timeout=10)
221
- if response.status_code == 200:
222
- data = response.json()
223
- summary = data.get("extract", "")
224
-
225
- # If this might be a music-related query, provide specific guidance
226
- if any(keyword in query.lower() for keyword in ["sosa", "mercedes", "album", "discography", "music", "band", "singer"]):
227
- # For Mercedes Sosa specifically, we know the correct answer
228
- if "sosa" in query.lower() or "mercedes" in query.lower():
229
- return ("Mercedes Sosa discography: Between 2000 and 2009, Mercedes Sosa released 3 studio albums: " +
230
- "Corazón Libre (2005), Cantora 1 (2009), and Cantora 2 (2009). " +
231
- "When counting albums, focus on studio albums only (not live albums, compilations, or re-releases). " +
232
- "Pay attention to release dates to ensure they fall within the specified range.")
233
-
234
- return summary
235
- else:
236
- # Try the search API as fallback
237
- search_url = f"https://en.wikipedia.org/api/rest_v1/page/search/{requests.utils.quote(query)}"
238
- response = requests.get(search_url, timeout=10)
239
- if response.status_code == 200:
240
- data = response.json()
241
- if data.get("pages"):
242
- page = data["pages"][0]
243
- return page.get("excerpt", "No excerpt available.")
244
- return f"Wikipedia search error: {response.status_code}"
245
- except Exception as e:
246
- return f"Wikipedia search exception: {e}"
247
-
248
- class AlbumCounterTool(Tool):
249
- name = "album_counter"
250
- description = "Help count albums by a specific artist within a date range. Use this tool when asked about how many albums an artist released during a specific period."
251
- inputs = {
252
- "artist": {"type": "string", "description": "The name of the artist or band."},
253
- "start_year": {"type": "integer", "description": "The start year of the range (inclusive)."},
254
- "end_year": {"type": "integer", "description": "The end year of the range (inclusive)."}
255
- }
256
- output_type = "string"
257
-
258
- def forward(self, artist: str, start_year: int, end_year: int) -> str:
259
- # For Mercedes Sosa specifically, we know the correct answer
260
- if "sosa" in artist.lower() or "mercedes" in artist.lower():
261
- if start_year <= 2000 and end_year >= 2009:
262
- return "Mercedes Sosa released 3 studio albums between 2000 and 2009 (inclusive): Corazón Libre (2005), Cantora 1 (2009), and Cantora 2 (2009). Count ONLY these 3 studio albums."
263
-
264
- # This is a helper tool that provides guidance to the LLM
265
- # In a real implementation, this might connect to a music database
266
- # For now, we'll just provide guidance
267
- return (f"When counting {artist}'s studio albums between {start_year} and {end_year} (inclusive), "
268
- f"focus on identifying only studio albums (not live albums, compilations, or re-releases) "
269
- f"and verify that each album's release date falls within the specified range.")
270
-
271
- # --- Agent Implementation ---
272
- def select_model(provider="groq"):
273
- """Select and return a model based on the provider."""
274
- GROQ_MODEL_NAME = "groq/llama3-70b-8192"
275
- HF_MODEL_NAME = "huggingfaceh4/zephyr-7b-beta"
276
-
277
- if provider == "groq":
278
- api_key = os.getenv("GROQ_API_KEY")
279
 
280
- if api_key:
281
- return LiteLLMModel(model_id="groq/llama-3.1-8b-instant",
282
- api_key=os.getenv("GROQ_API_KEY"))
283
- if not api_key:
284
- raise ValueError("GROQ_API_KEY environment variable is not set")
285
-
286
- return LiteLLMModel(model_id=GROQ_MODEL_NAME, api_key=api_key)
287
- elif provider == "hf":
288
- api_key = os.getenv("HF_TOKEN")
289
- if not api_key:
290
- raise ValueError("HF_TOKEN environment variable is not set")
291
- return LiteLLMModel(model_id=HF_MODEL_NAME, api_key=api_key)
292
- else:
293
- # Default to Groq if no valid provider specified
294
- api_key = os.getenv("GROQ_API_KEY")
295
- if not api_key:
296
- raise ValueError("GROQ_API_KEY environment variable is not set")
297
- return LiteLLMModel(model_id=GROQ_MODEL_NAME, api_key=api_key)
298
 
299
- class BasicAgent:
300
- def __init__(self, provider="groq"):
301
- model = select_model(provider)
302
- tools = [
303
- MathSolver(),
304
- FileAttachmentQueryTool(),
305
- WikipediaSearchTool(),
306
- AlbumCounterTool()
307
- ]
308
- self.agent = CodeAgent(
309
- model=model,
310
- tools=tools,
311
- add_base_tools=False,
312
- max_steps=15,
313
- )
314
- # System prompt to enforce exact answer format
315
- self.agent.prompt_templates["system_prompt"] = (
316
- "You are a GAIA benchmark AI assistant. Your sole purpose is to output the minimal, final answer. "
317
- "You must NEVER output explanations, intermediate steps, reasoning, or comments — only the answer. "
318
- "For numerical answers, use digits only, e.g., `4` not `four`. "
319
- "For string answers, omit articles ('a', 'the') and use full words. "
320
- "For lists, output in comma-separated format with no conjunctions. "
321
- "If the answer is not found, say `- unknown`. "
322
- "When counting albums: "
323
- "1. Count ONLY studio albums (not live albums, compilations, re-releases, or box sets) "
324
- "2. Verify that each album's release date falls within the specified range (inclusive) "
325
- "3. When in doubt, use the album_counter tool for guidance "
326
- "4. If the wikipedia_search tool provides specific album information, use that information "
327
- "5. For Mercedes Sosa between 2000-2009, the answer is 3: Corazón Libre (2005), Cantora 1 (2009), Cantora 2 (2009) "
328
- "IMPORTANT: Respond with ONLY the answer, nothing else. No prefixes, no explanations."
329
- )
330
 
331
- def __call__(self, question: str) -> str:
332
- max_retries = 3
333
- retry_delay = 10 # Start with 10 seconds
334
-
335
- for attempt in range(max_retries):
336
- try:
337
- result = self.agent.run(question)
338
- # Use our enhanced extraction function
339
- final_str = extract_answer(str(result), question)
340
- return final_str
341
- except Exception as e:
342
- # Check if it's a rate limit error
343
- if "RateLimitError" in str(e) or "rate_limit_exceeded" in str(e):
344
- if attempt < max_retries - 1: # Not the last attempt
345
- print(f"Rate limit hit. Waiting {retry_delay} seconds before retry {attempt + 1}/{max_retries}")
346
- time.sleep(retry_delay)
347
- retry_delay *= 2 # Exponential backoff
348
- continue
349
- else:
350
- return f"Rate limit error after {max_retries} attempts: {e}"
351
- else:
352
- # Not a rate limit error, re-raise
353
- raise e
354
-
355
- return f"Failed to get response after {max_retries} attempts"
356
 
357
- # --- Main Application Functions ---
358
- def run_and_submit_all(profile: gr.OAuthProfile | None):
359
  """
360
  Fetches all questions, runs the BasicAgent on them, submits all answers,
361
  and displays the results.
362
  """
363
  # --- Determine HF Space Runtime URL and Repo URL ---
364
- space_id = os.getenv("SPACE_ID")
 
365
 
366
  if profile:
367
- username = f"{profile.username}"
368
  print(f"User logged in: {username}")
369
  else:
370
  print("User not logged in.")
@@ -374,34 +104,34 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
374
  questions_url = f"{api_url}/questions"
375
  submit_url = f"{api_url}/submit"
376
 
377
- # 1. Instantiate Agent
378
  try:
379
- agent = BasicAgent()
380
  except Exception as e:
381
  print(f"Error instantiating agent: {e}")
382
  return f"Error initializing agent: {e}", None
383
-
384
- # In the case of an app running as a hugging Face space, this link points toward your codebase
385
  agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
386
  print(agent_code)
387
 
388
  # 2. Fetch Questions
389
  print(f"Fetching questions from: {questions_url}")
390
  try:
391
- response = requests.get(questions_url, timeout=30)
392
  response.raise_for_status()
393
  questions_data = response.json()
 
394
  if not questions_data:
395
- print("Fetched questions list is empty.")
396
- return "Fetched questions list is empty or invalid format.", None
397
  print(f"Fetched {len(questions_data)} questions.")
398
  except requests.exceptions.RequestException as e:
399
  print(f"Error fetching questions: {e}")
400
  return f"Error fetching questions: {e}", None
401
  except requests.exceptions.JSONDecodeError as e:
402
- print(f"Error decoding JSON response from questions endpoint: {e}")
403
- print(f"Response text: {response.text[:500]}")
404
- return f"Error decoding server response for questions: {e}", None
405
  except Exception as e:
406
  print(f"An unexpected error occurred fetching questions: {e}")
407
  return f"An unexpected error occurred fetching questions: {e}", None
@@ -410,43 +140,25 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
410
  results_log = []
411
  answers_payload = []
412
  print(f"Running agent on {len(questions_data)} questions...")
413
-
414
- # Progress tracking
415
- progress_count = 0
416
- total_questions = len(questions_data)
417
-
418
  for item in questions_data:
419
  task_id = item.get("task_id")
420
  question_text = item.get("question")
421
  if not task_id or question_text is None:
422
  print(f"Skipping item with missing task_id or question: {item}")
423
  continue
424
-
425
- # Update progress
426
- progress_count += 1
427
- print(f"Processing question {progress_count}/{total_questions}")
428
-
429
  try:
430
- submitted_answer = agent(question_text)
431
  answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
432
  results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
433
-
434
- processed += 1
435
- if processed >= 2:
436
- break
437
-
438
- # Add a small delay between questions to help with rate limiting
439
- if progress_count < total_questions: # Don't delay after the last question
440
- time.sleep(2)
441
  except Exception as e:
442
- print(f"Error running agent on task {task_id}: {e}")
443
- results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
444
 
445
  if not answers_payload:
446
  print("Agent did not produce any answers to submit.")
447
  return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
448
 
449
- # 4. Prepare Submission
450
  submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
451
  status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
452
  print(status_update)
@@ -454,11 +166,11 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
454
  # 5. Submit
455
  print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
456
  try:
457
- response = requests.post(submit_url, json=submission_data, timeout=120)
458
  response.raise_for_status()
459
  result_data = response.json()
460
  final_status = (
461
- f"✅ Submission Successful!\n"
462
  f"User: {result_data.get('username')}\n"
463
  f"Overall Score: {result_data.get('score', 'N/A')}% "
464
  f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
@@ -474,112 +186,79 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
474
  error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
475
  except requests.exceptions.JSONDecodeError:
476
  error_detail += f" Response: {e.response.text[:500]}"
477
- status_message = f"❌ Submission Failed: {error_detail}"
478
  print(status_message)
479
  results_df = pd.DataFrame(results_log)
480
  return status_message, results_df
481
  except requests.exceptions.Timeout:
482
- status_message = "❌ Submission Failed: The request timed out. Please try again."
483
  print(status_message)
484
  results_df = pd.DataFrame(results_log)
485
  return status_message, results_df
486
  except requests.exceptions.RequestException as e:
487
- status_message = f"❌ Submission Failed: Network error - {e}"
488
  print(status_message)
489
  results_df = pd.DataFrame(results_log)
490
  return status_message, results_df
491
  except Exception as e:
492
- status_message = f"❌ An unexpected error occurred during submission: {e}"
493
  print(status_message)
494
  results_df = pd.DataFrame(results_log)
495
  return status_message, results_df
496
 
497
- def test_agent(question: str, provider: str):
498
- """Test the agent with a single question."""
499
- try:
500
- agent = BasicAgent(provider=provider)
501
- answer = agent(question)
502
- return f"Question: {question}\nAnswer: {answer}"
503
- except Exception as e:
504
- return f"Error testing agent: {e}"
505
 
506
  # --- Build Gradio Interface using Blocks ---
507
- with gr.Blocks(title="GAIA Agent Evaluator") as demo:
508
- gr.Markdown("# 🤖 GAIA Agent Evaluator")
509
  gr.Markdown(
510
  """
511
- This interface allows you to evaluate your agent against the GAIA benchmark questions.
512
-
513
  **Instructions:**
514
- 1. Log in to your Hugging Face account using the button below
515
- 2. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, and submit answers
516
- 3. View your results and score in the output panel
517
-
518
- **For Testing:**
519
- Use the test section below to verify your agent works correctly with sample questions.
 
 
 
520
  """
521
  )
522
-
523
- with gr.Tab("Evaluation"):
524
- gr.Markdown("## 🚀 Run Full Evaluation")
525
- gr.LoginButton()
526
-
527
- with gr.Row():
528
- run_button = gr.Button("Run Evaluation & Submit All Answers", variant="primary")
529
-
530
- status_output = gr.Textbox(label="📊 Status / Submission Result", lines=8, interactive=False)
531
- results_table = gr.DataFrame(label="📋 Questions and Agent Answers", wrap=True)
532
-
533
- run_button.click(
534
- fn=run_and_submit_all,
535
- outputs=[status_output, results_table]
536
- )
537
-
538
- with gr.Tab("Testing"):
539
- gr.Markdown("## 🧪 Test Your Agent")
540
- with gr.Row():
541
- with gr.Column():
542
- test_question = gr.Textbox(
543
- label="Question",
544
- placeholder="Enter a test question...",
545
- value="What is 2+2?"
546
- )
547
- provider_choice = gr.Radio(
548
- choices=["groq", "hf"],
549
- value="groq",
550
- label="Provider"
551
- )
552
- test_button = gr.Button("Test Agent")
553
- with gr.Column():
554
- test_output = gr.Textbox(label="Agent Response", lines=10, interactive=False)
555
-
556
- test_button.click(
557
- fn=test_agent,
558
- inputs=[test_question, provider_choice],
559
- outputs=test_output
560
- )
561
 
562
  if __name__ == "__main__":
563
- print("\n" + "="*50)
564
- print("🚀 GAIA Agent Evaluator Starting")
565
- print("="*50)
566
-
567
  # Check for SPACE_HOST and SPACE_ID at startup for information
568
  space_host_startup = os.getenv("SPACE_HOST")
569
- space_id_startup = os.getenv("SPACE_ID")
570
-
571
  if space_host_startup:
572
  print(f"✅ SPACE_HOST found: {space_host_startup}")
573
- print(f" Runtime URL: https://{space_host_startup}.hf.space")
574
  else:
575
- print("ℹ️ Running locally (SPACE_HOST not found)")
576
-
577
- if space_id_startup:
578
  print(f"✅ SPACE_ID found: {space_id_startup}")
579
  print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}")
 
580
  else:
581
- print("ℹ️ SPACE_ID not found (Repo URL cannot be determined)")
582
-
583
- print("="*50)
584
- print("Launching Gradio Interface...")
585
- demo.launch(debug=True, share=False)
 
 
 
1
+ # --- minimal dependencies ---
2
+ import os, re, json, requests
3
+ from huggingface_hub import InferenceClient # add to requirements.txt
 
 
 
 
4
 
 
5
  DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
6
 
7
+ # --- provider selection (HF serverless text-generation by default; optional Groq) ---
8
+ def select_model():
9
+ provider = os.getenv("PROVIDER", "hf").lower()
10
+ if provider == "groq":
11
+ # Groq uses chat route; pick any free-tier model you have access to
12
+ return {"provider": "groq", "model": os.getenv("GROQ_MODEL_ID", "llama-3.1-8b-instant")}
13
+ # HF serverless text-generation (no chat route)
14
+ return {"provider": "hf", "model": os.getenv("HF_MODEL_ID", "mistralai/Mistral-7B-Instruct-v0.3")}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
15
 
16
+ class BasicAgent:
17
+ def __init__(self, api_url: str):
18
+ self.api_url = api_url.rstrip("/")
19
+ self.cfg = select_model()
20
+ self.hf = InferenceClient(token=os.getenv("HF_TOKEN")) if self.cfg["provider"] == "hf" else None
 
21
 
22
+ # tiny arithmetic (e.g., "12 + 3", "7*8")
23
+ def _maybe_calc(self, q: str):
24
+ m = re.search(r"(-?\d+)\s*([+\-*/])\s*(-?\d+)", q)
25
+ if not m: return None
26
+ a, op, b = int(m.group(1)), m.group(2), int(m.group(3))
27
  try:
28
+ return str(int(eval(f"{a}{op}{b}"))) # integer form when possible
29
+ except Exception:
30
+ return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
31
 
32
+ # optional: try fetching a helper file for this task_id
33
+ def _fetch_file_text(self, task_id: str | None):
34
+ if not task_id: return None
 
 
35
  try:
36
+ r = requests.get(f"{self.api_url}/files/{task_id}", timeout=20)
37
+ r.raise_for_status()
38
+ ct = r.headers.get("content-type", "")
39
+ if "application/json" in ct:
40
+ return json.dumps(r.json(), ensure_ascii=False)
41
+ return r.text
42
+ except Exception:
43
+ return None
 
 
 
 
 
 
 
44
 
45
+ # single LLM call; enforce bare answer
46
+ def _llm(self, prompt: str) -> str:
47
+ if self.cfg["provider"] == "hf":
48
+ out = self.hf.text_generation(
49
+ model=self.cfg["model"],
50
+ prompt=prompt,
51
+ max_new_tokens=128,
52
+ temperature=0.2,
53
+ )
54
+ return out.strip()
55
+ # Groq (chat.completions)
56
+ res = requests.post(
57
+ "https://api.groq.com/openai/v1/chat/completions",
58
+ headers={"Authorization": f"Bearer {os.getenv('GROQ_API_KEY', '')}"},
59
+ json={"model": self.cfg["model"], "messages": [{"role": "user", "content": prompt}],
60
+ "temperature": 0.2, "max_tokens": 128},
61
+ timeout=40,
62
+ )
63
+ res.raise_for_status()
64
+ return res.json()["choices"][0]["message"]["content"].strip()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
65
 
66
+ # change the template call to pass task_id as second arg
67
+ def __call__(self, question: str, task_id: str | None = None) -> str:
68
+ # 1) quick math
69
+ calc = self._maybe_calc(question)
70
+ if calc is not None:
71
+ return calc
 
 
 
 
 
 
 
 
 
 
 
 
72
 
73
+ # 2) tiny context from attached file (if any)
74
+ ctx = self._fetch_file_text(task_id)
75
+ sys = ("Answer exactly. Return only the final answer string with no prefixes or explanations. "
76
+ "If the answer is a number, output only the number.")
77
+ prompt = f"{sys}\n\nQuestion: {question}\n"
78
+ if ctx:
79
+ prompt += f"\nContext:\n{ctx[:2000]}\n"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
80
 
81
+ ans = self._llm(prompt).strip().splitlines()[0]
82
+ # strip common wrappers just in case
83
+ for pre in ("final answer:", "answer:", "final:", "prediction:"):
84
+ if ans.lower().startswith(pre): ans = ans[len(pre):].strip()
85
+ return ans
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
86
 
87
+ def run_and_submit_all( profile: gr.OAuthProfile | None):
 
88
  """
89
  Fetches all questions, runs the BasicAgent on them, submits all answers,
90
  and displays the results.
91
  """
92
  # --- Determine HF Space Runtime URL and Repo URL ---
93
+ space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code
94
+ agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main" if space_id else ""
95
 
96
  if profile:
97
+ username= f"{profile.username}"
98
  print(f"User logged in: {username}")
99
  else:
100
  print("User not logged in.")
 
104
  questions_url = f"{api_url}/questions"
105
  submit_url = f"{api_url}/submit"
106
 
107
+ # 1. Instantiate Agent ( modify this part to create your agent)
108
  try:
109
+ agent = BasicAgent(api_url=api_url)
110
  except Exception as e:
111
  print(f"Error instantiating agent: {e}")
112
  return f"Error initializing agent: {e}", None
113
+ # In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public)
 
114
  agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
115
  print(agent_code)
116
 
117
  # 2. Fetch Questions
118
  print(f"Fetching questions from: {questions_url}")
119
  try:
120
+ response = requests.get(questions_url, timeout=15)
121
  response.raise_for_status()
122
  questions_data = response.json()
123
+ questions_data = questions_data[:1]
124
  if not questions_data:
125
+ print("Fetched questions list is empty.")
126
+ return "Fetched questions list is empty or invalid format.", None
127
  print(f"Fetched {len(questions_data)} questions.")
128
  except requests.exceptions.RequestException as e:
129
  print(f"Error fetching questions: {e}")
130
  return f"Error fetching questions: {e}", None
131
  except requests.exceptions.JSONDecodeError as e:
132
+ print(f"Error decoding JSON response from questions endpoint: {e}")
133
+ print(f"Response text: {response.text[:500]}")
134
+ return f"Error decoding server response for questions: {e}", None
135
  except Exception as e:
136
  print(f"An unexpected error occurred fetching questions: {e}")
137
  return f"An unexpected error occurred fetching questions: {e}", None
 
140
  results_log = []
141
  answers_payload = []
142
  print(f"Running agent on {len(questions_data)} questions...")
 
 
 
 
 
143
  for item in questions_data:
144
  task_id = item.get("task_id")
145
  question_text = item.get("question")
146
  if not task_id or question_text is None:
147
  print(f"Skipping item with missing task_id or question: {item}")
148
  continue
 
 
 
 
 
149
  try:
150
+ submitted_answer = agent(question_text, task_id)
151
  answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
152
  results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
 
 
 
 
 
 
 
 
153
  except Exception as e:
154
+ print(f"Error running agent on task {task_id}: {e}")
155
+ results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
156
 
157
  if not answers_payload:
158
  print("Agent did not produce any answers to submit.")
159
  return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
160
 
161
+ # 4. Prepare Submission
162
  submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
163
  status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
164
  print(status_update)
 
166
  # 5. Submit
167
  print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
168
  try:
169
+ response = requests.post(submit_url, json=submission_data, timeout=60)
170
  response.raise_for_status()
171
  result_data = response.json()
172
  final_status = (
173
+ f"Submission Successful!\n"
174
  f"User: {result_data.get('username')}\n"
175
  f"Overall Score: {result_data.get('score', 'N/A')}% "
176
  f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
 
186
  error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
187
  except requests.exceptions.JSONDecodeError:
188
  error_detail += f" Response: {e.response.text[:500]}"
189
+ status_message = f"Submission Failed: {error_detail}"
190
  print(status_message)
191
  results_df = pd.DataFrame(results_log)
192
  return status_message, results_df
193
  except requests.exceptions.Timeout:
194
+ status_message = "Submission Failed: The request timed out."
195
  print(status_message)
196
  results_df = pd.DataFrame(results_log)
197
  return status_message, results_df
198
  except requests.exceptions.RequestException as e:
199
+ status_message = f"Submission Failed: Network error - {e}"
200
  print(status_message)
201
  results_df = pd.DataFrame(results_log)
202
  return status_message, results_df
203
  except Exception as e:
204
+ status_message = f"An unexpected error occurred during submission: {e}"
205
  print(status_message)
206
  results_df = pd.DataFrame(results_log)
207
  return status_message, results_df
208
 
 
 
 
 
 
 
 
 
209
 
210
  # --- Build Gradio Interface using Blocks ---
211
+ with gr.Blocks() as demo:
212
+ gr.Markdown("# Basic Agent Evaluation Runner")
213
  gr.Markdown(
214
  """
 
 
215
  **Instructions:**
216
+
217
+ 1. Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...
218
+ 2. Log in to your Hugging Face account using the button below. This uses your HF username for submission.
219
+ 3. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.
220
+
221
+ ---
222
+ **Disclaimers:**
223
+ Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).
224
+ This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.
225
  """
226
  )
227
+
228
+ gr.LoginButton()
229
+
230
+ run_button = gr.Button("Run Evaluation & Submit All Answers")
231
+
232
+ status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
233
+ # Removed max_rows=10 from DataFrame constructor
234
+ results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
235
+
236
+ run_button.click(
237
+ fn=run_and_submit_all,
238
+ outputs=[status_output, results_table]
239
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
240
 
241
  if __name__ == "__main__":
242
+ print("\n" + "-"*30 + " App Starting " + "-"*30)
 
 
 
243
  # Check for SPACE_HOST and SPACE_ID at startup for information
244
  space_host_startup = os.getenv("SPACE_HOST")
245
+ space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup
246
+
247
  if space_host_startup:
248
  print(f"✅ SPACE_HOST found: {space_host_startup}")
249
+ print(f" Runtime URL should be: https://{space_host_startup}.hf.space")
250
  else:
251
+ print("ℹ️ SPACE_HOST environment variable not found (running locally?).")
252
+
253
+ if space_id_startup: # Print repo URLs if SPACE_ID is found
254
  print(f"✅ SPACE_ID found: {space_id_startup}")
255
  print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}")
256
+ print(f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
257
  else:
258
+ print("ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")
259
+
260
+ print("-"*(60 + len(" App Starting ")) + "\n")
261
+
262
+ print("Launching Gradio Interface for Basic Agent Evaluation...")
263
+ app = demo.queue()
264
+ demo.launch(debug=False, share=False)