Intro Tutorial Part 2 not working with together.ai (ChatTogether()), resulting in a 504: Gateway time-out

[HaupChris]

I am doing this intro Tutorial at the moment and I am running into an Issue when trying to use the ChatTogether class for LLM usage.
In the first part of the intro everything is fine. But when I adjust the code to as explained in the second part to use the Tavily Search tool, the method graph.stream() results in a Error 502. I have set the necessary environment variables for the code to run and when I use ChatOpenAI instead of ChatTogether it does work es expected. Does anyone know what the reason could be?

This is the code:

import json
import os

from dotenv import load_dotenv
from langchain_core.messages import BaseMessage, ToolMessage
from langchain_openai import ChatOpenAI
from langchain_together import ChatTogether
from typing import Annotated
from typing_extensions import TypedDict
from typing import Literal
from IPython.display import Image, display

from langgraph.graph import StateGraph
from langgraph.graph.message import add_messages
from langchain_community.tools.tavily_search import TavilySearchResults

env_path = ... # left out here
load_dotenv(env_path)

class State(TypedDict):
    # Messages have the type "list". The `add_messages` function
    # in the annotation defines how this state key should be updated
    # (in this case, it appends messages to the list, rather than overwriting them)
    messages: Annotated[list, add_messages]

llm = ChatTogether(model="meta-llama/Llama-3-8b-chat-hf", request_timeout=1000)
# llm = ChatOpenAI(model="gpt-3.5-turbo", request_timeout=1000)

tool = TavilySearchResults(max_results=2)
tools = [tool]
tool.invoke("What's a 'node' in LangGraph?")

llm_with_tools = llm.bind_tools(tools)


def chatbot(state: State):
    return {"messages": [llm_with_tools.invoke(state["messages"])]}
    
class BasicToolNode:
    """A node that runs the tools requested in the last AIMessage."""

    def __init__(self, tools: list) -> None:
        self.tools_by_name = {tool.name: tool for tool in tools}

    def __call__(self, inputs: dict):
        if messages := inputs.get("messages", []):
            message = messages[-1]
        else:
            raise ValueError("No message found in input")
        outputs = []
        for tool_call in message.tool_calls:
            tool_result = self.tools_by_name[tool_call["name"]].invoke(
                tool_call["args"]
            )
            outputs.append(
                ToolMessage(
                    content=json.dumps(tool_result),
                    name=tool_call["name"],
                    tool_call_id=tool_call["id"],
                )
            )
        return {"messages": outputs}

def route_tools(
    state: State,
) -> Literal["action", "__end__"]:
    """Use in the conditional_edge to route to the ToolNode if the last message

    has tool calls. Otherwise, route to the end."""
    if isinstance(state, list):
        ai_message = state[-1]
    elif messages := state.get("messages", []):
        ai_message = messages[-1]
    else:
        raise ValueError(f"No messages found in input state to tool_edge: {state}")
    if hasattr(ai_message, "tool_calls") and len(ai_message.tool_calls) > 0:
        return "action"
    return "__end__"

tool_node = BasicToolNode(tools=[tool])

graph_builder = StateGraph(State)

graph_builder.add_node("action", tool_node)
graph_builder.add_node("chatbot", chatbot)


# The `tools_condition` function returns "action" if the chatbot asks to use a tool, and "__end__" if
# it is fine directly responding. This conditional routing defines the main agent loop.
graph_builder.add_conditional_edges(
    "chatbot",
    route_tools,
    # The following dictionary lets you tell the graph to interpret the condition's outputs as a specific node
    # It defaults to the identity function, but if you
    # want to use a node named something else apart from "action",
    # You can update the value of the dictionary to something else
    # e.g., "action": "my_tools"
    {"action": "action", "__end__": "__end__"},
)
# Any time a tool is called, we return to the chatbot to decide the next step
graph_builder.add_edge("action", "chatbot")
graph_builder.set_entry_point("chatbot")
graph = graph_builder.compile()    


try:
    display(Image(graph.get_graph().draw_mermaid_png()))
except:
    # This requires some extra dependencies and is optional
    pass
    
from langchain_core.messages import BaseMessage

while True:
    user_input = input("User: ")
    if user_input.lower() in ["quit", "exit", "q"]:
        print("Goodbye!")
        break
    for event in graph.stream({"messages": [("user", user_input)]}):
        for value in event.values():
            if isinstance(value["messages"][-1], BaseMessage):
                print("Assistant:", value["messages"][-1].content)    

When I run the loop, there occurs an InternalServerError:

InternalServerError: <!DOCTYPE html>
<!--[if lt IE 7]> <html class="no-js ie6 oldie" lang="en-US"> <![endif]-->
<!--[if IE 7]>    <html class="no-js ie7 oldie" lang="en-US"> <![endif]-->
<!--[if IE 8]>    <html class="no-js ie8 oldie" lang="en-US"> <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en-US"> <!--<![endif]-->
<head>


<title>api.together.ai | 504: Gateway time-out</title>
<meta charset="UTF-8" />
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=Edge" />
<meta name="robots" content="noindex, nofollow" />
<meta name="viewport" content="width=device-width,initial-scale=1" />
<link rel="stylesheet" id="cf_styles-css" href="/cdn-cgi/styles/main.css" />

[hinthornw]

Hmm this seems like a Together issue more than a langgraph one - Could you share a langsmith trace for this so we could help debug?

[cmazzoni87]

Unfortunately no. The issue is persistent and only present when using langchain_together's ChatTogether within Langgraph.
https://smith.langchain.com/public/1c5d4796-14e3-49eb-a0db-cd0f78c76ee9/r

[hinthornw]

Hm I can't view the link - was it unshared? Will try to help if you give me information to debug.