xleven / ask-sspai

QA on Sspai Documents

Hi there 👋

Greetings from xleven!

I’m right now a full stack AI engineer who loves LLMs and all cutting-edge AI techs. Formerly I worked as a data scientist for several years.

Projects I've been building recently:

• AI Chatroom: Multiple AI bots in a chatroom while you take control
• Agenty: Team up with AI agents
• AI Hackathon Judge: An AI judge for daily LLM hackathons
• DataVizQA: QA on your data with visualizations

Blogs where I write about AI, data, tech and all:

• xleven's blog (English)
• 万千十一的博客 (中文)

ask-sspai's Issues

Implementing Langchain handlers

Hey there @xleven!!

Curious if you would be able to help me with implementing the langchain callbacks using RunnableSequence. The code I am using is from Jacob Lee's (langchain maintainer) langchain vector store example.

I am trying to figure out how to add the callbacks needed for the langchain handler where I upsret into my supabase db

Thanks!

import { getSession } from '@/app/supabase-server';
import { Database } from '@/types_db';
import { PineconeClient } from '@pinecone-database/pinecone';
import { createRouteHandlerClient } from '@supabase/auth-helpers-nextjs';
import { Message as VercelChatMessage, StreamingTextResponse, LangChainStream } from 'ai';
import { nanoid } from '@/lib/utils';
import { ChatOpenAI } from 'langchain/chat_models/openai';
import { Document } from 'langchain/document';
import { OpenAIEmbeddings } from 'langchain/embeddings/openai';
import { PromptTemplate } from 'langchain/prompts';
import {
  BytesOutputParser,
  StringOutputParser
} from 'langchain/schema/output_parser';
import {
  RunnableSequence,
  RunnablePassthrough
} from 'langchain/schema/runnable';
import { PineconeStore } from 'langchain/vectorstores/pinecone';
import { SupabaseVectorStore } from 'langchain/vectorstores/supabase';
import { cookies } from 'next/headers';
import { NextRequest, NextResponse } from 'next/server';
import { Configuration, OpenAIApi } from 'openai-edge';

export const runtime = 'edge';

const configuration = new Configuration({
  apiKey: process.env.OPENAI_API_KEY
});

const openai = new OpenAIApi(configuration);

const combineDocumentsFn = (docs: Document[], separator = '\n\n') => {
  const serializedDocs = docs.map((doc) => doc.pageContent);
  return serializedDocs.join(separator);
};

const formatVercelMessages = (chatHistory: VercelChatMessage[]) => {
  const formattedDialogueTurns = chatHistory.map((message) => {
    if (message.role === 'user') {
      return `Human: ${message.content}`;
    } else if (message.role === 'assistant') {
      return `Assistant: ${message.content}`;
    } else {
      return `${message.role}: ${message.content}`;
    }
  });
  return formattedDialogueTurns.join('\n');
};

const CONDENSE_QUESTION_TEMPLATE = `Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question, in its original language.

<chat_history>
  {chat_history}
</chat_history>

Follow Up Input: {question}
Standalone question:`;
const condenseQuestionPrompt = PromptTemplate.fromTemplate(
  CONDENSE_QUESTION_TEMPLATE
);

const ANSWER_TEMPLATE = `You are a bird, reply like a little bird would reply

Answer the question based only on the following context and chat history:
<context>
  {context}
</context>

<chat_history>
  {chat_history}
</chat_history>

Question: {question}
`;
const answerPrompt = PromptTemplate.fromTemplate(ANSWER_TEMPLATE);

/**
 * This handler initializes and calls a retrieval chain. It composes the chain using
 * LangChain Expression Language. See the docs for more information:
 *
 * https://js.langchain.com/docs/guides/expression_language/cookbook#conversational-retrieval-chain
 */
export async function POST(req: NextRequest) {
  const cookieStore = cookies();
  const supabase = createRouteHandlerClient<Database>({
    cookies: () => cookieStore
  });
  const session = await getSession();
  const userId = session?.user.id;

  if (!userId) {
    return new Response('Unauthorized', {
      status: 401
    });
  }
  try {
    const json = await req.json();
    const messages = json.messages ?? [];
    const previousMessages = messages.slice(0, -1);
    const currentMessageContent = messages[messages.length - 1].content;

    const model = new ChatOpenAI({
      modelName: 'gpt-3.5-turbo',
      temperature: 0.2
    });

    const pinecone = new PineconeClient();

    await pinecone.init({
      environment: process.env.PINECONE_ENVIRONMENT ?? '',
      apiKey: process.env.PINECONE_API_KEY ?? ''
    });

    const pineconeIndex = pinecone.Index(process.env.PINECONE_INDEX_NAME!);
    const vectorstore = await PineconeStore.fromExistingIndex(
      new OpenAIEmbeddings(),
      { pineconeIndex }
    );

    /**
     * We use LangChain Expression Language to compose two chains.
     * To learn more, see the guide here:
     *
     * https://js.langchain.com/docs/guides/expression_language/cookbook
     */

    // const { handlers } = LangChainStream({
    //   async onCompletion(completion) {
    //     const title = json.messages[0].content.substring(0, 100);
    //     const id = json.id ?? nanoid();
    //     const createdAt = Date.now();
    //     const path = `/chat/${id}`;
    //     const payload = {
    //       id,
    //       title,
    //       userId,
    //       createdAt,
    //       path,
    //       messages: [
    //         ...messages,
    //         {
    //           content: completion,
    //           role: 'assistant'
    //         }
    //       ]
    //     };
    //     await supabase.from('chats').upsert({ id, payload }).throwOnError();
    //   }
    // });

    const standaloneQuestionChain = RunnableSequence.from([
      condenseQuestionPrompt,
      model,
      new StringOutputParser()
    ]);

    let resolveWithDocuments: (value: Document[]) => void;
    const documentPromise = new Promise<Document[]>((resolve) => {
      resolveWithDocuments = resolve;
    });

    const retriever = vectorstore.asRetriever({
      callbacks: [
        {
          handleRetrieverEnd(documents) {
            resolveWithDocuments(documents);
          }
        }
      ]
    });

    const retrievalChain = retriever.pipe(combineDocumentsFn);

    const answerChain = RunnableSequence.from([
      {
        context: RunnableSequence.from([
          (input) => input.question,
          retrievalChain
        ]),
        chat_history: (input) => input.chat_history,
        question: (input) => input.question
      },
      answerPrompt,
      model
    ]);

    const conversationalRetrievalQAChain = RunnableSequence.from([
      {
        question: standaloneQuestionChain,
        chat_history: (input) => input.chat_history
      },
      answerChain,
      new BytesOutputParser()
    ]);

    const stream = await conversationalRetrievalQAChain.stream({
      question: currentMessageContent,
      chat_history: formatVercelMessages(previousMessages)
    });

    const documents = await documentPromise;
    const serializedSources = Buffer.from(
      JSON.stringify(
        documents.map((doc) => {
          return {
            pageContent: doc.pageContent.slice(0, 50) + '...',
            metadata: doc.metadata
          };
        })
      )
    ).toString('base64');

    return new StreamingTextResponse(stream, {
      headers: {
        'x-message-index': (previousMessages.length + 1).toString(),
        'x-sources': serializedSources
      }
    });
  } catch (e: any) {
    return NextResponse.json({ error: e.message }, { status: 500 });
  }
}