Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 21 additions & 2 deletions adapter/openai/processor.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,27 @@ func getChoices(form *ChatStreamResponse) *globals.Chunk {

choice := form.Choices[0].Delta

content := choice.Content

// handle gemini multi-modal inline data (base64 image)
if len(content) == 0 && choice.MultiModal != nil {
for _, item := range *choice.MultiModal {
if item.InlineData == nil || len(item.InlineData.Data) == 0 {
continue
}

mime := item.InlineData.MimeType
if len(mime) == 0 {
mime = "image/png"
}
// use markdown image to let frontend直接渲染而不是显示base64文本
content = fmt.Sprintf("![image](data:%s;base64,%s)", mime, item.InlineData.Data)
break
}
}

return &globals.Chunk{
Content: choice.Content,
Content: content,
ToolCall: choice.ToolCalls,
FunctionCall: choice.FunctionCall,
}
Expand Down Expand Up @@ -128,7 +147,7 @@ func (c *ChatInstance) ProcessLine(data string, isCompletionType bool) (*globals
}

if form := processChatErrorResponse(data); form != nil {
return &globals.Chunk{Content: ""}, errors.New(fmt.Sprintf("openai error: %s (type: %s)", form.Error.Message, form.Error.Type))
return &globals.Chunk{Content: ""}, fmt.Errorf("openai error: %s (type: %s)", form.Error.Message, form.Error.Type)
}

globals.Warn(fmt.Sprintf("openai error: cannot parse chat completion response: %s", data))
Expand Down
12 changes: 12 additions & 0 deletions globals/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ type Message struct {
ToolCallId *string `json:"tool_call_id,omitempty"` // only `tool` role
ToolCalls *ToolCalls `json:"tool_calls,omitempty"` // only `assistant` role
ReasoningContent *string `json:"reasoning_content,omitempty"` // only for deepseek reasoner models
MultiModal *MultiModal `json:"multi_mod_content,omitempty"` // gemini inline data (e.g., image base64)
}

type Chunk struct {
Expand All @@ -18,6 +19,17 @@ type Chunk struct {
FunctionCall *FunctionCall `json:"function_call,omitempty"`
}

type InlineData struct {
Data string `json:"data"`
MimeType string `json:"mime_type"`
}

type MultiModalContent struct {
InlineData *InlineData `json:"inline_data,omitempty"`
}

type MultiModal []MultiModalContent

type ChatSegmentResponse struct {
Conversation int64 `json:"conversation"`
Quota float32 `json:"quota"`
Expand Down
14 changes: 14 additions & 0 deletions utils/scanner.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,10 @@ func EventScanner(props *EventScannerProps, config ...globals.ProxyConfig) *Even
}

func processFullSSE(body io.ReadCloser, callback func(string) error) *EventScannerError {
// allow large SSE lines (e.g., image/base64 payloads)
scanner := bufio.NewScanner(body)
buf := make([]byte, 0, 1024*1024)
scanner.Buffer(buf, 16*1024*1024)
var eventType, eventData string
var buffer strings.Builder

Expand Down Expand Up @@ -164,11 +167,18 @@ func processFullSSE(body io.ReadCloser, callback func(string) error) *EventScann
}
}

if err := scanner.Err(); err != nil {
return &EventScannerError{Error: err}
}

return nil
}

func processLegacySSE(body io.ReadCloser, callback func(string) error) *EventScannerError {
// default scanner buffer (64kb) is not enough for image/base64 chunks
scanner := bufio.NewScanner(body)
buf := make([]byte, 0, 1024*1024)
scanner.Buffer(buf, 16*1024*1024)
scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 {
// when EOF and empty data
Expand Down Expand Up @@ -219,5 +229,9 @@ func processLegacySSE(body io.ReadCloser, callback func(string) error) *EventSca
}
}

if err := scanner.Err(); err != nil {
return &EventScannerError{Error: err}
}

return nil
}