langchain multi modal support
How to pass multimodal data directly to models
https://python.langchain.com/v0.2/docs/how_to/multimodal_inputs/
message = HumanMessage( content=[ {"type": "text", "text": "describe the weather in this image"}, {"type": "image_url", "image_url": {"url": image_url}}, ], ) response = model.invoke([message]) print(response.content)
message = HumanMessage( content=[ {"type": "text", "text": "describe the weather in this image"}, { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_data}"}, }, ], ) response = model.invoke([message]) print(response.content)
from typing import Literal from langchain_core.tools import tool @tool def weather_tool(weather: Literal["sunny", "cloudy", "rainy"]) -> None: """Describe the weather""" pass model_with_tools = model.bind_tools([weather_tool]) message = HumanMessage( content=[ {"type": "text", "text": "describe the weather in this image"}, {"type": "image_url", "image_url": {"url": image_url}}, ], ) response = model_with_tools.invoke([message]) print(response.tool_calls)
How to use ImagePromptTemplate
https://github.com/langchain-ai/langchain/discussions/20820
from langchain.agents import AgentExecutor, create_openai_tools_agent from langchain.schema import SystemMessage from langchain_community.tools import CopyFileTool from langchain_core.prompts import ( ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, ) from langchain_openai import ChatOpenAI # Define the prompt template with the image_url prompt_messages = [ SystemMessage(content="Describe the image provided"), HumanMessagePromptTemplate.from_template( template=[ {"type": "image_url", "image_url": {"url": "{image_url}"}}, ] ), MessagesPlaceholder("agent_scratchpad"), ] prompt = ChatPromptTemplate(messages=prompt_messages) # Define the LLM model llm = ChatOpenAI(model="gpt-4o") # Define a dummy tool tools = [CopyFileTool()] # Create an Agent that can be used to call the tools we defined agent = create_openai_tools_agent(llm, tools, prompt) # Define the AgentExecutor properly agent_executor = AgentExecutor( agent=agent, tools=tools, ) # Example image URL you want to pass in at runtime image_url = "https://fastly.picsum.photos/id/450/200/200.jpg?hmac=DluUYibC-zBoNHLOHsO6aHIuiA3pDhholFjiR5KcwR0" # Execute the Agent with the image URL response = agent_executor.invoke({"image_url": image_url, "agent_scratchpad": []})
Image-caption-with-LLM-and-Langchain
https://github.com/stemgene/Image-caption-with-LLM-and-Langchain/blob/main/main.py
import streamlit as st import os import requests import torch from PIL import Image from langchain.agents import initialize_agent from langchain.chat_models import ChatOpenAI from langchain.chains.conversation.memory import ConversationBufferWindowMemory from tools import ImageCaptionTool, ObjectDetectionTool from tempfile import NamedTemporaryFile ######################## ### Initialize agent ### ######################## tools = [ImageCaptionTool(), ObjectDetectionTool()] conversational_memory = ConversationBufferWindowMemory( memory_key='chat_history', k=5, return_messages=True ) llm = ChatOpenAI( # openai_api_key=os.environ.get("OPENAI_API_KEY"), # for local usage openai_api_key=st.secrets['auth_key'], temperature=0, model_name='gpt-3.5-turbo' ) agent = initialize_agent( agent='chat-conversational-react-description', tools=tools, llm=llm, max_iterations=5, verbose=True, memory=conversational_memory, early_stoppy_method='generate' ) # Set title st.title("Ask a question to an image") # set header st.header("Please upload an image") # upload file # file = st.file_uploader("", type=['jpg', 'png', 'jpeg']) # input image url image_url = st.text_input('Please input the image URL') if image_url and image_url != "": image_object = Image.open(requests.get(image_url, stream=True).raw).convert("RGB") # display image #st.image(file, use_column_width=True) st.image(image_object, use_column_width=True) # text input user_question = st.text_input('Ask a question your image:') ############################## ### Compute agent response ### ############################## # with NamedTemporaryFile(dir='.', mode='w+b') as f: # f.write(file.getbuffer()) # image_path = f.name # # write agent response # if user_question and user_question != "": # with st.spinner(text='In progress...'): # response = agent.run('{}, this is the image path: {}'.format(user_question, image_path)) # st.write(response) #write agent response if user_question and user_question != "": with st.spinner(text='In progress...'): response = agent.run('{}, this is the image path: {}'.format(user_question, image_url)) st.write(response)
how_to_use_crew_ai_for_image_understanding
https://github.com/mdwoicke/crewai_multi_agent/blob/main/01_how_to_use_crew_ai_for_image_understanding.py
""" Author: Rajib Deb Date: 02/10/2024 Description: This is the driver program that starts the MasterChef crew. """ from crew_ai_crews.master_chef import MasterChefCrew if __name__ == "__main__": # url="https://rumkisgoldenspoon.com/wp-content/uploads/2022/02/Aar-macher-jhol.jpg" url="https://m.media-amazon.com/images/I/51dFvTRE3iL.__AC_SX300_SY300_QL70_FMwebp_.jpg" crew = MasterChefCrew(url=url) print(crew.kickoff())
分析菜肴成分，并给出做法
""" Author: Rajib Deb Date: 02/10/2024 Description: This module creates the crew objects with all the required agents and tasks """ import os from crewai import Crew from dotenv import load_dotenv from crew_ai_crews.agents import MasterChef from crew_ai_crews.tasks import ExtractIngredientsFromImage load_dotenv() OPENAI_API_KEY = os.environ.get('OPENAI_API_KEY') class MasterChefCrew(): """ The master chef crew consisting of the food connoisseur and the chef """ def __init__(self, url): agents = MasterChef(url=url) self.extract_agent = agents.get_recipe_from_image() self.instruct_agent = agents.instruct_to_cook_dish() def kickoff(self): """ Creating the crew with the two agents :return: """ tasks = ExtractIngredientsFromImage() crew = Crew( agents=[self.extract_agent, self.instruct_agent], tasks=[tasks.get_ingredients(self.extract_agent), tasks.cooking_instruction(self.instruct_agent)], verbose=True ) result = crew.kickoff() return result
LangChain vs. CrewAI: Comparing AI Development Platforms
https://smythos.com/ai-agents/ai-agent-builders/langchain-vs-crewai/
crewai不支持多模态， 其本身是workflow定义
Feature Comparison Table
LangChain CrewAI SmythOS CORE FEATURES Hosted Agents (Dev, Production) ✅ ❌ ✅ Environments (Dev, Production) ✅ ❌ ✅ Visual Builder ❌ ❌ ✅ No-Code Options ❌ ❌ ✅ Explainability & Transparency ✅ ❌ ✅ Debug Tools ✅ ❌ ✅ Multimodal ✅ ❌ ✅ Audit Logs for Analytics ✅ ❌ ✅ Agent Work Scheduler ❌ ✅ ✅ SECURITY Constrained Alignment ❌ ❌ ✅ Data Encryption ✅ ❌ ✅ OAuth ✅ ❌ ✅ IP Control ❌ ❌ ✅ COMPONENTS Foundation AIs ✅ ❌ ✅ Huggingface AIs ✅ ❌ ✅ Zapier APIs ❌ ❌ ✅ All other APIs, RPA ✅ ❌ ✅ Classifiers ✅ ❌ ✅ Logic ✅ ❌ ✅ Data Lakes ❌ ❌ ✅ DEPLOYMENT OPTIONS (EMBODIMENTS) Deploy as API ✅ ❌ ✅ Deploy as Webhook ❌ ❌ ✅ Staging Domains ❌ ❌ ✅ Production Domains ❌ ❌ ✅ API Authentication (OAuth + Key) ✅ ❌ ✅ Deploy as Site Chat ✅ ❌ ✅ Deploy as Scheduled Agent ❌ ❌ ✅ Deploy as GPT ✅ ❌ ✅ DATA LAKE SUPPORT Hosted Vector Database ❌ ❌ ✅ Sitemap Crawler ❌ ❌ ✅ YouTube Transcript Crawler ❌ ❌ ✅ URL Crawler ❌ ❌ ✅ PDF Support ✅ ❌ ✅ Word File Support ❌ ❌ ✅ TXT File Support ✅ ❌ ✅
Multi-modal outputs: Image & Text
https://python.langchain.com.cn/docs/use_cases/agents/multi_modal_output_agent
audio理解
https://js.langchain.com/v0.2/docs/how_to/tool_calls_multimodal/
https://js.langchain.ac.cn/v0.2/docs/how_to/tool_calls_multimodal/
import { SystemMessage } from "@langchain/core/messages";
import { tool } from "@langchain/core/tools";
const summaryTool = tool(
(input) => {
return input.summary;
},
{
name: "summary_tool",
description: "Log the summary of the content",
schema: z.object({
summary: z.string().describe("The summary of the content to log"),
}),
}
);
const audioUrl =
"https://www.pacdv.com/sounds/people_sound_effects/applause-1.wav";
const axiosRes = await axios.get(audioUrl, { responseType: "arraybuffer" });
const base64 = btoa(
new Uint8Array(axiosRes.data).reduce(
(data, byte) => data + String.fromCharCode(byte),
""
)
);
const model = new ChatGoogleGenerativeAI({
model: "gemini-1.5-pro-latest",
}).bindTools([summaryTool]);
const response = await model.invoke([
new SystemMessage(
"Summarize this content. always use the summary_tool in your response"
),
new HumanMessage({
content: [
{
type: "media",
mimeType: "audio/wav",
data: base64,
},
],
}),
]);
console.log(response.tool_calls);
How to use multimodal prompts
https://python.langchain.com/docs/how_to/multimodal_prompts/
https://blog.csdn.net/cheese_burger_/article/details/140351968
prompt = ChatPromptTemplate.from_messages( [ ("system", "compare the two pictures provided"), ( "user", [ { "type": "image_url", "image_url": {"url": "data:image/jpeg;base64,{image_data1}"}, }, { "type": "image_url", "image_url": {"url": "data:image/jpeg;base64,{image_data2}"}, }, ], ), ] )
快速封装替换自定义LLM（基于自定义API或本地模型）
https://blog.csdn.net/HYY_2000/article/details/138339333
LangGraph: Multi-Agent Workflows
https://blog.langchain.dev/langgraph-multi-agent-workflows/
Other Frameworks
LangGraph is not the first framework to support multi-agent workflows. Most of the difference between these frameworks largely lies in the mental models and concepts they introduce.
Autogen
Autogen emerged as perhaps the first multi-agent framework. The biggest difference in mental model between LangGraph and Autogen is in construction of the agents. LangGraph prefers an approach where you explicitly define different agents and transition probabilities, preferring to represent it as a graph. Autogen frames it more as a "conversation". We believe that this "graph" framing makes it more intuitive and provides better developer experience for constructing more complex and opinionated workflows where you really want to control the transition probabilities between nodes. It also supports workflows that aren't explicitly captured by "conversations."
Another key difference between Autogen and LangGraph is that LangGraph is fully integrated into the LangChain ecosystem, meaning you take fully advantage of all the LangChain integrations and LangSmith observability.
CrewAI
Another key framework we want to highlight is CrewAI. CrewAI has emerged recently as a popular way to create multi-agent "teams". Compared to LangGraph, CrewAI is a higher-level framework. In fact, we are actively working with the CrewAI team to integration LangGraph into CrewAI! We think CrewAI has arrived at an awesome higher level DevEx and we want to support that.
CrewAI Unleashed: Future of AI Agent Teams
https://blog.langchain.dev/crewai-unleashed-future-of-ai-agent-teams/
CrewAI represents a shift in AI agents by offering a thin framework that leverages collaboration and roleplaying, based on versatility and efficiency. It stands as a tool for engineers and creatives alike, enabling the seamless assembly of AI agents into cohesive, high-performing teams.
Whether it's transforming a single thought into a fully-fledged landing page or conducting complex idea analysis, CrewAI is adept at handling a diverse range of tasks through its processes.
The real-world applications of CrewAI, from boosting social media presence to building interactive landing pages, underscore its practicality and adaptability. Looking forward, CrewAI is set to evolve further, introducing more intricate processes and continuing to redefine the landscape of AI teamwork. With its user-friendly integration and customization options, CrewAI is not just a concept but a tangible, powerful tool to harness the power of AI Agents.
