Skip to content

Commit f0a1a79

Browse files
shyamnamboodiripadshyamnamboodiripad
authored
Display images in conversation for report (#6286)
Also includes some fixes / improvements around how images are processed in the Safety evaluators. * Avoid redundant / double base64 encoding for images in `DataContnet` * Check for supported image extensions * Add tests for `DataContent` images
1 parent dfdf2c5 commit f0a1a79

File tree

11 files changed

+194
-57
lines changed

11 files changed

+194
-57
lines changed

‎src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ConversationDetails.tsx‎

Lines changed: 43 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,7 @@ import { useState } from "react";
44
import ReactMarkdown from "react-markdown";
55
import { useReportContext } from "./ReportContext";
66
import { useStyles } from "./Styles";
7-
import { ChatMessageDisplay } from "./Summary";
8-
7+
import { ChatMessageDisplay, isTextContent, isImageContent } from "./Summary";
98

109
export const ConversationDetails = ({ messages, model, usage }: {
1110
messages: ChatMessageDisplay[];
@@ -25,6 +24,40 @@ export const ConversationDetails = ({ messages, model, usage }: {
2524
usage?.totalTokenCount && `Total Tokens: ${usage.totalTokenCount}`,
2625
].filter(Boolean).join(' • ');
2726

27+
const renderContent = (content: AIContent) => {
28+
if (isTextContent(content)) {
29+
return renderMarkdown ?
30+
<ReactMarkdown>{content.text}</ReactMarkdown> :
31+
<pre className={classes.preWrap}>{content.text}</pre>;
32+
} else if (isImageContent(content)) {
33+
const imageUrl = (content as UriContent).uri || (content as DataContent).uri;
34+
return <img src={imageUrl} alt="Content" className={classes.imageContent} />;
35+
}
36+
};
37+
38+
const groupMessages = () => {
39+
const result: { role: string, participantName: string, contents: AIContent[] }[] = [];
40+
41+
for (const message of messages) {
42+
// If this message has the same role and participant as the previous one, append its content
43+
const lastGroup = result[result.length - 1];
44+
if (lastGroup && lastGroup.role === message.role && lastGroup.participantName === message.participantName) {
45+
lastGroup.contents.push(message.content);
46+
} else {
47+
// Otherwise, start a new group
48+
result.push({
49+
role: message.role,
50+
participantName: message.participantName,
51+
contents: [message.content]
52+
});
53+
}
54+
}
55+
56+
return result;
57+
};
58+
59+
const messageGroups = groupMessages();
60+
2861
return (
2962
<div className={classes.section}>
3063
<div className={classes.sectionHeader} onClick={() => setIsExpanded(!isExpanded)}>
@@ -35,20 +68,22 @@ export const ConversationDetails = ({ messages, model, usage }: {
3568

3669
{isExpanded && (
3770
<div className={classes.sectionContainer}>
38-
{messages.map((message, index) => {
39-
const isFromUserSide = isUserSide(message.role);
71+
{messageGroups.map((group, index) => {
72+
const isFromUserSide = isUserSide(group.role);
4073
const messageRowClass = mergeClasses(
4174
classes.messageRow,
4275
isFromUserSide ? classes.userMessageRow : classes.assistantMessageRow
4376
);
4477

4578
return (
4679
<div key={index} className={messageRowClass}>
47-
<div className={classes.messageParticipantName}>{message.participantName}</div>
80+
<div className={classes.messageParticipantName}>{group.participantName}</div>
4881
<div className={classes.messageBubble}>
49-
{renderMarkdown ?
50-
<ReactMarkdown>{message.content}</ReactMarkdown> :
51-
<pre className={classes.preWrap}>{message.content}</pre>}
82+
{group.contents.map((content, contentIndex) => (
83+
<div key={contentIndex}>
84+
{renderContent(content)}
85+
</div>
86+
))}
5287
</div>
5388
</div>
5489
);

‎src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/EvalTypes.d.ts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,12 +54,23 @@ type AIContent = {
5454
$type: string;
5555
};
5656

57-
// TODO: Model other types of AIContent such as function calls, function call results, images, audio etc.
57+
// TODO: Model other types of AIContent such as function calls, function call results, audio etc.
5858
type TextContent = AIContent & {
5959
$type: "text";
6060
text: string;
6161
};
6262

63+
type UriContent = AIContent & {
64+
$type: "uri";
65+
uri: string;
66+
mediaType: string;
67+
};
68+
69+
type DataContent = AIContent & {
70+
$type: "data";
71+
uri: string;
72+
};
73+
6374
type EvaluationResult = {
6475
metrics: {
6576
[K: string]: MetricWithNoValue | NumericMetric | BooleanMetric | StringMetric;

‎src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Styles.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,10 @@ export const useStyles = makeStyles({
205205
preWrap: {
206206
whiteSpace: 'pre-wrap',
207207
},
208+
imageContent: {
209+
maxWidth: '100%',
210+
maxHeight: '400px',
211+
},
208212
executionHeaderCell: {
209213
display: 'flex',
210214
alignItems: 'center',

‎src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Summary.ts‎

Lines changed: 37 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -108,9 +108,15 @@ export class ScoreNode {
108108
const { messages } = getConversationDisplay(lastMessage ? [lastMessage] : [], this.scenario?.modelResponse);
109109
let history = "";
110110
if (messages.length === 1) {
111-
history = messages[0].content;
111+
const content = messages[0].content;
112+
if (isTextContent(content)) {
113+
history = content.text;
114+
}
112115
} else if (messages.length > 1) {
113-
history = messages.map(m => `[${m.participantName}] ${m.content}`).join("\n\n");
116+
history = messages
117+
.filter(m => isTextContent(m.content))
118+
.map(m => `[${m.participantName}] ${(m.content as TextContent).text}`)
119+
.join("\n\n");
114120
}
115121

116122
this.shortenedPrompt = shortenPrompt(history);
@@ -284,10 +290,25 @@ const flattener = function* (node: ScoreNode): Iterable<ScoreNode> {
284290
}
285291
};
286292

287-
const isTextContent = (content: AIContent): content is TextContent => {
293+
export const isTextContent = (content: AIContent): content is TextContent => {
288294
return (content as TextContent).text !== undefined;
289295
};
290296

297+
export const isImageContent = (content: AIContent): content is UriContent | DataContent => {
298+
if ((content as UriContent).uri !== undefined && (content as UriContent).mediaType) {
299+
return (content as UriContent).mediaType.startsWith("image/");
300+
}
301+
302+
if ((content as DataContent).uri !== undefined) {
303+
const dataContent = content as DataContent;
304+
if (dataContent.uri.startsWith('data:image/')) {
305+
return true;
306+
}
307+
}
308+
309+
return false;
310+
};
311+
291312
export type ConversationDisplay = {
292313
messages: ChatMessageDisplay[];
293314
model?: string;
@@ -297,36 +318,32 @@ export type ConversationDisplay = {
297318
export type ChatMessageDisplay = {
298319
role: string;
299320
participantName: string;
300-
content: string;
321+
content: AIContent;
301322
};
302323

303324
export const getConversationDisplay = (messages: ChatMessage[], modelResponse?: ChatResponse): ConversationDisplay => {
304325
const chatMessages: ChatMessageDisplay[] = [];
305326

306327
for (const m of messages) {
307328
for (const c of m.contents) {
308-
if (isTextContent(c)) {
309-
const participantName = m.authorName ? `${m.authorName} (${m.role})` : m.role;
310-
chatMessages.push({
311-
role: m.role,
312-
participantName: participantName,
313-
content: c.text
314-
});
315-
}
329+
const participantName = m.authorName ? `${m.authorName} (${m.role})` : m.role;
330+
chatMessages.push({
331+
role: m.role,
332+
participantName: participantName,
333+
content: c
334+
});
316335
}
317336
}
318337

319338
if (modelResponse?.messages) {
320339
for (const m of modelResponse.messages) {
321340
for (const c of m.contents) {
322-
if (isTextContent(c)) {
323-
const participantName = m.authorName ? `${m.authorName} (${m.role})` : m.role || 'Assistant';
324-
chatMessages.push({
325-
role: m.role,
326-
participantName: participantName,
327-
content: c.text
328-
});
329-
}
341+
const participantName = m.authorName ? `${m.authorName} (${m.role})` : m.role || 'Assistant';
342+
chatMessages.push({
343+
role: m.role,
344+
participantName: participantName,
345+
content: c
346+
});
330347
}
331348
}
332349
}

‎src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System;
5+
6+
namespace Microsoft.Extensions.AI.Evaluation.Safety;
7+
internal static class AIContentExtensions
8+
{
9+
internal static bool IsTextOrUsage(this AIContent content)
10+
=> content is TextContent || content is UsageContent;
11+
12+
internal static bool IsImageWithSupportedFormat(this AIContent content) =>
13+
(content is UriContent uriContent && IsSupportedImageFormat(uriContent.MediaType)) ||
14+
(content is DataContent dataContent && IsSupportedImageFormat(dataContent.MediaType));
15+
16+
internal static bool IsUriBase64Encoded(this DataContent dataContent)
17+
{
18+
ReadOnlyMemory<char> uri = dataContent.Uri.AsMemory();
19+
20+
int commaIndex = uri.Span.IndexOf(',');
21+
if (commaIndex == -1)
22+
{
23+
return false;
24+
}
25+
26+
ReadOnlyMemory<char> metadata = uri.Slice(0, commaIndex);
27+
28+
bool isBase64Encoded = metadata.Span.EndsWith(";base64".AsSpan(), StringComparison.OrdinalIgnoreCase);
29+
return isBase64Encoded;
30+
}
31+
32+
private static bool IsSupportedImageFormat(string mediaType)
33+
{
34+
// 'image/jpeg' is the official MIME type for JPEG. However, some systems recognize 'image/jpg' as well.
35+
36+
return
37+
mediaType.Equals("image/jpeg", StringComparison.OrdinalIgnoreCase) ||
38+
mediaType.Equals("image/jpg", StringComparison.OrdinalIgnoreCase) ||
39+
mediaType.Equals("image/png", StringComparison.OrdinalIgnoreCase) ||
40+
mediaType.Equals("image/gif", StringComparison.OrdinalIgnoreCase);
41+
}
42+
}

‎src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatMessageExtensions.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System.Collections.Generic;
5+
using System.Linq;
6+
7+
namespace Microsoft.Extensions.AI.Evaluation.Safety;
8+
9+
internal static class ChatMessageExtensions
10+
{
11+
internal static bool ContainsImageWithSupportedFormat(this ChatMessage message)
12+
=> message.Contents.Any(c => c.IsImageWithSupportedFormat());
13+
14+
internal static bool ContainsImageWithSupportedFormat(this IEnumerable<ChatMessage> conversation)
15+
=> conversation.Any(ContainsImageWithSupportedFormat);
16+
}

‎src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatResponseExtensions.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
namespace Microsoft.Extensions.AI.Evaluation.Safety;
5+
6+
internal static class ChatResponseExtensions
7+
{
8+
internal static bool ContainsImageWithSupportedFormat(this ChatResponse response)
9+
=> response.Messages.ContainsImageWithSupportedFormat();
10+
}

‎src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs‎

Lines changed: 14 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -12,19 +12,6 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety;
1212

1313
internal static class ContentSafetyServicePayloadUtilities
1414
{
15-
internal static bool IsImage(this AIContent content) =>
16-
(content is UriContent uriContent && uriContent.HasTopLevelMediaType("image")) ||
17-
(content is DataContent dataContent && dataContent.HasTopLevelMediaType("image"));
18-
19-
internal static bool ContainsImage(this ChatMessage message)
20-
=> message.Contents.Any(IsImage);
21-
22-
internal static bool ContainsImage(this ChatResponse response)
23-
=> response.Messages.ContainsImage();
24-
25-
internal static bool ContainsImage(this IEnumerable<ChatMessage> conversation)
26-
=> conversation.Any(ContainsImage);
27-
2815
internal static (string payload, IReadOnlyList<EvaluationDiagnostic>? diagnostics) GetPayload(
2916
ContentSafetyServicePayloadFormat payloadFormat,
3017
IEnumerable<ChatMessage> conversation,
@@ -356,16 +343,25 @@ IEnumerable<JsonObject> GetContents(ChatMessage message)
356343
}
357344
else if (content is DataContent dataContent && dataContent.HasTopLevelMediaType("image"))
358345
{
359-
BinaryData imageBytes = BinaryData.FromBytes(dataContent.Data);
360-
string base64ImageData = Convert.ToBase64String(imageBytes.ToArray());
346+
string url;
347+
if (dataContent.IsUriBase64Encoded())
348+
{
349+
url = dataContent.Uri;
350+
}
351+
else
352+
{
353+
BinaryData imageBytes = BinaryData.FromBytes(dataContent.Data);
354+
string base64ImageData = Convert.ToBase64String(imageBytes.ToArray());
355+
url = $"data:{dataContent.MediaType};base64,{base64ImageData}";
356+
}
361357

362358
yield return new JsonObject
363359
{
364360
["type"] = "image_url",
365361
["image_url"] =
366362
new JsonObject
367363
{
368-
["url"] = $"data:{dataContent.MediaType};base64,{base64ImageData}"
364+
["url"] = url
369365
}
370366
};
371367
}
@@ -475,7 +471,7 @@ void ValidateContents(ChatMessage message)
475471

476472
if (areImagesSupported)
477473
{
478-
if (content.IsImage())
474+
if (content.IsImageWithSupportedFormat())
479475
{
480476
++imagesCount;
481477
}
@@ -533,7 +529,7 @@ void ValidateContents(ChatMessage message)
533529
EvaluationDiagnostic.Warning(
534530
$"The supplied conversation contained {unsupportedContentCount} instances of unsupported content within messages. " +
535531
$"The current evaluation being performed by {evaluatorName} only supports content of type '{nameof(TextContent)}', '{nameof(UriContent)}' and '{nameof(DataContent)}'. " +
536-
$"For '{nameof(UriContent)}' and '{nameof(DataContent)}', only content with media type 'image/*' is supported. " +
532+
$"For '{nameof(UriContent)}' and '{nameof(DataContent)}', only content with media type 'image/png', 'image/jpeg' and 'image/gif' are supported. " +
537533
$"The unsupported contents were ignored for this evaluation."));
538534
}
539535
else
@@ -582,7 +578,4 @@ void ValidateContents(ChatMessage message)
582578

583579
return (turns, normalizedPerTurnContext, diagnostics, contentType);
584580
}
585-
586-
private static bool IsTextOrUsage(this AIContent content)
587-
=> content is TextContent || content is UsageContent;
588581
}

‎src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ProtectedMaterialEvaluator.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ await EvaluateContentSafetyAsync(
8787

8888
// If images are present in the conversation, do a second evaluation for protected material in images.
8989
// The content safety service does not support evaluating both text and images in the same request currently.
90-
if (messages.ContainsImage() || modelResponse.ContainsImage())
90+
if (messages.ContainsImageWithSupportedFormat() || modelResponse.ContainsImageWithSupportedFormat())
9191
{
9292
EvaluationResult imageResult =
9393
await EvaluateContentSafetyAsync(

‎test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/Microsoft.Extensions.AI.Evaluation.Integration.Tests.csproj‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,18 @@
22

33
<PropertyGroup>
44
<TargetFrameworks>$(LatestTargetFramework)</TargetFrameworks>
5-
<RootNamespace>Microsoft.Extensions.AI.Evaluation.Integration.Tests</RootNamespace>
5+
<RootNamespace>Microsoft.Extensions.AI</RootNamespace>
66
<Description>Integration tests for Microsoft.Extensions.AI.Evaluation.</Description>
77
</PropertyGroup>
8+
9+
<ItemGroup>
10+
<EmbeddedResource Include="..\..\Shared\ImageDataUri\dotnet.png" Link="Resources\dotnet.png"/>
11+
</ItemGroup>
12+
13+
<ItemGroup>
14+
<Compile Include="..\..\Shared\ImageDataUri\ImageDataUri.cs" Link="Shared\ImageDataUri\ImageDataUri.cs" />
15+
</ItemGroup>
16+
817

918
<ItemGroup>
1019
<PackageReference Include="Azure.AI.OpenAI" />

0 commit comments

Comments
 (0)