Multimodal Support

Fresh

Mem0 supports multimodal data including images and documents. When users submit visual or document content, the system extracts textual information and adds it to user memory.

Supported Media Types

Type	Formats
Images	JPG, PNG, common image formats
Documents	MDX, TXT
PDFs	PDF files via URL

Image Integration

URL-Based Images

image_url = "https://example.com/photo.jpg"

image_message = {
    "role": "user",
    "content": {
        "type": "image_url",
        "image_url": {
            "url": image_url
        }
    }
}
client.add([image_message], user_id="alice")

Base64-Encoded Images

Python

import base64

image_path = "path/to/your/image.jpg"

with open(image_path, "rb") as image_file:
    base64_image = base64.b64encode(image_file.read()).decode("utf-8")

image_message = {
    "role": "user",
    "content": {
        "type": "image_url",
        "image_url": {
            "url": f"data:image/jpeg;base64,{base64_image}"
        }
    }
}
client.add([image_message], user_id="alice")

TypeScript

import MemoryClient from "mem0ai";
import fs from 'fs';

const imagePath = 'path/to/your/image.jpg';
const base64Image = fs.readFileSync(imagePath, { encoding: 'base64' });

const imageMessage = {
    role: "user",
    content: {
        type: "image_url",
        image_url: {
            url: `data:image/jpeg;base64,${base64Image}`
        }
    }
};

await client.add([imageMessage], { user_id: "alice" });

Text Document Integration

URL-Based Documents

document_url = "https://example.com/document.txt"

document_message = {
    "role": "user",
    "content": {
        "type": "mdx_url",
        "mdx_url": {
            "url": document_url
        }
    }
}
client.add([document_message], user_id="alice")

Base64-Encoded Documents

import base64

document_path = "path/to/your/document.txt"

def file_to_base64(file_path):
    with open(file_path, "rb") as file:
        return base64.b64encode(file.read()).decode('utf-8')

base64_document = file_to_base64(document_path)

document_message = {
    "role": "user",
    "content": {
        "type": "mdx_url",
        "mdx_url": {
            "url": base64_document
        }
    }
}
client.add([document_message], user_id="alice")

PDF Documents

pdf_url = "https://example.com/report.pdf"

pdf_message = {
    "role": "user",
    "content": {
        "type": "pdf_url",
        "pdf_url": {
            "url": pdf_url
        }
    }
}
client.add([pdf_message], user_id="alice")

Complete Multi-Type Example

import base64
from mem0 import MemoryClient

client = MemoryClient()

# Image URL
image_msg = {
    "role": "user",
    "content": {
        "type": "image_url",
        "image_url": {"url": "https://example.com/photo.jpg"}
    }
}

# Text document URL
text_msg = {
    "role": "user",
    "content": {
        "type": "mdx_url",
        "mdx_url": {"url": "https://example.com/doc.txt"}
    }
}

# PDF URL
pdf_msg = {
    "role": "user",
    "content": {
        "type": "pdf_url",
        "pdf_url": {"url": "https://example.com/report.pdf"}
    }
}

client.add([image_msg], user_id="alice")
client.add([text_msg], user_id="alice")
client.add([pdf_msg], user_id="alice")