Working version: I am using Groq API keys and mock API specs.
import asyncio
import json
import logging
import os
from dotenv import load_dotenv
from llama_index.core.agent import ReActAgent
from llama_index.core.memory import ChatMemoryBuffer
from llama_index.core.tools import FunctionTool
from llama_index.llms.groq import Groq
load_dotenv()
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('multi-tool-reproduction')
# Mock data to simulate the OpenAPI + Requests workflow
MOCK_API_SPECS = {
"companies": {
"endpoint": "/v1/companies/list",
"method": "POST",
"base_url": "https://api.my-company.com",
"description": "List all companies for authenticated user"
},
"users": {
"endpoint": "/v1/users/profile",
"method": "GET",
"base_url": "https://api.my-company.com",
"description": "Get user profile information"
}
}
MOCK_API_RESPONSES = {
"https://api.my-company.com/v1/companies/list": {
"success": True,
"companies": [
{"id": 1, "name": "Acme Corp", "status": "active"},
{"id": 2, "name": "Tech Solutions Inc", "status": "active"},
{"id": 3, "name": "Global Enterprises", "status": "inactive"}
]
},
"https://api.example.com/companies": {
"success": False,
"error": "Invalid domain - this is the wrong endpoint!"
}
}
def mock_load_openapi_spec(query: str) -> str:
"""
Mock version of OpenAPIToolSpec functionality
This simulates finding API endpoints based on user queries
"""
logger.info(f"🔍 OPENAPI TOOL CALLED with query: '{query}'")
query_lower = query.lower()
# Simple matching logic
if "companies" in query_lower or "list" in query_lower:
spec = MOCK_API_SPECS["companies"]
result = {
"found": True,
"endpoint": spec["endpoint"],
"method": spec["method"],
"full_url": f"{spec['base_url']}{spec['endpoint']}",
"description": spec["description"],
"base_url": spec["base_url"]
}
logger.info(f"📋 OPENAPI FOUND: {spec['base_url']}{spec['endpoint']}")
elif "users" in query_lower or "profile" in query_lower:
spec = MOCK_API_SPECS["users"]
result = {
"found": True,
"endpoint": spec["endpoint"],
"method": spec["method"],
"full_url": f"{spec['base_url']}{spec['endpoint']}",
"description": spec["description"],
"base_url": spec["base_url"]
}
logger.info(f"📋 OPENAPI FOUND: {spec['base_url']}{spec['endpoint']}")
else:
result = {
"found": False,
"error": f"No API endpoint found for query: {query}",
"suggestion": "Try queries like 'list companies' or 'get user profile'"
}
logger.info("📋 OPENAPI: No matching endpoint found")
return json.dumps(result, indent=2)
def mock_post_request(url: str, body: str = "{}", headers: str = "{}") -> str:
"""
Mock version of RequestsToolSpec post_request functionality
This simulates making HTTP POST requests
"""
logger.info(f"🌐 HTTP POST TOOL CALLED with URL: '{url}'")
try:
request_body = json.loads(body) if body else {}
request_headers = json.loads(headers) if headers else {}
# Mock response based on URL
if url in MOCK_API_RESPONSES:
response_data = MOCK_API_RESPONSES[url]
logger.info(f"📡 HTTP SUCCESS: Found mock response for {url}")
else:
# This simulates the BUG - when wrong URL is used
response_data = {
"success": False,
"error": f"No mock response for URL: {url}",
"message": "This represents the bug - agent used wrong URL!",
"expected_urls": list(MOCK_API_RESPONSES.keys())
}
logger.warning(f"📡 HTTP FAILURE: No mock response for {url}")
result = {
"status_code": 200 if response_data.get("success") else 400,
"url": url,
"request_body": request_body,
"request_headers": request_headers,
"response": response_data
}
return json.dumps(result, indent=2)
except Exception as e:
logger.error(f"❌ HTTP tool error: {e}")
return json.dumps({
"success": False,
"error": str(e),
"url": url
})
def mock_get_request(url: str, headers: str = "{}") -> str:
"""
Mock version of RequestsToolSpec get_request functionality
"""
logger.info(f"🌐 HTTP GET TOOL CALLED with URL: '{url}'")
try:
request_headers = json.loads(headers) if headers else {}
if url in MOCK_API_RESPONSES:
response_data = MOCK_API_RESPONSES[url]
logger.info(f"📡 HTTP SUCCESS: Found mock response for {url}")
else:
response_data = {
"success": False,
"error": f"No mock response for URL: {url}",
"message": "This represents the bug - agent used wrong URL!"
}
logger.warning(f"📡 HTTP FAILURE: No mock response for {url}")
result = {
"status_code": 200 if response_data.get("success") else 400,
"url": url,
"request_headers": request_headers,
"response": response_data
}
return json.dumps(result, indent=2)
except Exception as e:
logger.error(f"❌ HTTP GET tool error: {e}")
return json.dumps({
"success": False,
"error": str(e),
"url": url
})
async def test_multi_tool_agent():
"""Test agent with both tools - THE MAIN ISSUE"""
print("\n" + "=" * 80)
print("🧪 TEST 3: MULTI-TOOL AGENT (THE BUG)")
print("=" * 80)
try:
# Create both tools
openapi_tool = FunctionTool.from_defaults(
fn=mock_load_openapi_spec,
name="load_openapi_spec",
description="Find API endpoints and specifications based on user query. Returns JSON with endpoint details including the full URL to use."
)
http_tool = FunctionTool.from_defaults(
fn=mock_post_request,
name="post_request",
description="Make HTTP POST requests to API endpoints. Requires the full URL including domain name."
)
# System prompt similar to Stack Overflow issue
system_prompt = """
You are an API assistant with two tools: load_openapi_spec and post_request.
IMPORTANT WORKFLOW:
- FIRST: Use load_openapi_spec to find the correct API endpoint for the user's request
- SECOND: Use post_request with the EXACT full URL from the first tool's response
Always follow this two-step process. Never guess URLs or use default endpoints.
"""
memory = ChatMemoryBuffer.from_defaults()
llm = Groq(model="llama3-70b-8192", api_key=os.getenv("GROQ_API_KEY"))
agent = ReActAgent.from_tools(
tools=[openapi_tool, http_tool],
llm=llm,
memory=memory,
verbose=True,
system_prompt=system_prompt
)
print("\n🚀 Running test...")
response = await agent.achat("List my companies")
print(f"\n📋 Final response: {response}")
except Exception as e:
print(f"❌ Multi-tool agent error: {e}")
logger.exception("Multi-tool agent detailed error:")
async def main():
"""Run all tests to reproduce the multi-tool chaining issue"""
print("LlamaIndex Multi-Tool Chaining")
print("=" * 60)
# Test multi-tool agent (the main issue)
await test_multi_tool_agent()
print("\n" + "=" * 80)
if __name__ == "__main__":
asyncio.run(main())
Answer:
🚀 Running test...
> Running step 1a1b21a8-5b87-4379-94b2-941769dfeedc. Step input: List my companies
INFO:httpx:HTTP Request: POST https://api.groq.com/openai/v1/chat/completions "HTTP/1.1 200 OK"
Thought: The current language of the user is: English. I need to use a tool to help me answer the question.
Action: load_openapi_spec
Action Input: {'query': 'list companies'}
INFO:multi-tool-reproduction:🔍 OPENAPI TOOL CALLED with query: 'list companies'
INFO:multi-tool-reproduction:📋 OPENAPI FOUND: https://api.my-company.com/v1/companies/list
Observation: {
"found": true,
"endpoint": "/v1/companies/list",
"method": "POST",
"full_url": "https://api.my-company.com/v1/companies/list",
"description": "List all companies for authenticated user",
"base_url": "https://api.my-company.com"
}
> Running step 853cc07c-b6f7-4d1b-9ae6-21006cc7b731. Step input: None
INFO:httpx:HTTP Request: POST https://api.groq.com/openai/v1/chat/completions "HTTP/1.1 200 OK"
Thought: I have the API endpoint to list companies. Now I need to make a POST request to this endpoint.
Action: post_request
Action Input: {'url': 'https://api.my-company.com/v1/companies/list', 'body': '{}', 'headers': '{}'}
INFO:multi-tool-reproduction:🌐 HTTP POST TOOL CALLED with URL: 'https://api.my-company.com/v1/companies/list'
INFO:multi-tool-reproduction:📡 HTTP SUCCESS: Found mock response for https://api.my-company.com/v1/companies/list
Observation: {
"status_code": 200,
"url": "https://api.my-company.com/v1/companies/list",
"request_body": {},
"request_headers": {},
"response": {
"success": true,
"companies": [
{
"id": 1,
"name": "Acme Corp",
"status": "active"
},
{
"id": 2,
"name": "Tech Solutions Inc",
"status": "active"
},
{
"id": 3,
"name": "Global Enterprises",
"status": "inactive"
}
]
}
}
> Running step 35b8297b-e908-47f9-9356-08de6505cad7. Step input: None
INFO:httpx:HTTP Request: POST https://api.groq.com/openai/v1/chat/completions "HTTP/1.1 200 OK"
Thought: I have the list of companies. I can answer the user's question now.
Answer: Here is the list of your companies: Acme Corp, Tech Solutions Inc, Global Enterprises
📋 Final response: Here is the list of your companies: Acme Corp, Tech Solutions Inc, Global Enterprises