Skip to content

Release #9

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fengb3 merged 2 commits into from
Mar 11, 2025
Merged

Release #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7ebaf57
fix upload document for knowledgebase with file
fengb3 Mar 11, 2025
1b60958
Update version to 0.0.1-alpha3
fengb3 Mar 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 73 additions & 7 deletions DifySharp.Demo.AspNet/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using DifySharp;
using DifySharp.Chat.ChatMessages;
using DifySharp.Extensions;
using DifySharp.KnowledgeBase;
using DifySharp.KnowledgeBase.Document;
using WebApiClientCore.Parameters;

var builder = WebApplication.CreateBuilder(args);

Expand All @@ -17,13 +20,18 @@

var app = builder.Build();

app.MapGet("/", async (IServiceProvider sp) =>
var group = app.MapGroup("example");
var knowledgeGroup = group.MapGroup("knowledge");

knowledgeGroup.MapGet("create_file_by_text", async (IServiceProvider sp) =>
{
var api = sp.GetRequiredKeyedService<KnowledgeBaseClient>("knowledge"); // get client instance by name in configuration
var api = sp
.GetRequiredKeyedService<KnowledgeBaseClient>("knowledge"); // get client instance by name in configuration

var uuid = Guid.NewGuid().ToString("N")[..6];

var response = await api.PostCreateDocumentByTextAsync("", // add a dataset id here
var response = await api.PostCreateDocumentByTextAsync(
"<your-dataset-id>", // add a dataset id here
new CreateByText.RequestBody(
$"Test Document {uuid}",
"Test Content",
Expand Down Expand Up @@ -78,16 +86,74 @@
};
});

knowledgeGroup.MapGet("create_file_by_file", async (IServiceProvider sp) =>
{
var api = sp
.GetRequiredKeyedService<KnowledgeBaseClient>("knowledge"); // get client instance by name in configuration

var uuid = Guid.NewGuid().ToString("N")[..6];

var tmpFile = Path.GetTempFileName();
await File.WriteAllTextAsync(tmpFile, "Test Content");

var _defaultProcessRule = new ProcessRule(
"automatic",
new Rules(
[
new PreProcessingRule(
"remove_extra_spaces",
true
),
new PreProcessingRule(
"remove_urls_emails",
true
)
],
new Segmentation(
"\n\n",
1000
),
"paragraph",
new SubChunkSegmentation(
"\n\n",
1000,
200
)
)
);

// var dataJsonStr =
// "{\"indexing_technique\":\"high_quality\",\"process_rule\":{\"rules\":{\"pre_processing_rules\":[{\"id\":\"remove_extra_spaces\",\"enabled\":true},{\"id\":\"remove_urls_emails\",\"enabled\":true}],\"segmentation\":{\"separator\":\"###\",\"max_tokens\":500}},\"mode\":\"custom\"}}";

var data = new CreateByFile.Data(
null,
IndexingTechnique.Economy,
DocForm.TextModel,
"",
_defaultProcessRule
);

var response = await api.PostCreateDocumentByFileAsync( // add a dataset id here
"<your-dataset-id>", data, new FormDataFile("<your-file-path>"));

var document = response.Document;

return new
{
document
};
});

app.MapGet("/ChatApiDemo/ChatMessagesBlocking", async (IServiceProvider sp) =>
{
// get chat client instance by name in configuration
var client = sp.GetRequiredKeyedService<ChatClient>("chat");
var client = sp.GetRequiredKeyedService<ChatClient>("chat");

// send chat message in blocking mode
var response = await client.PostChatMessageBlocking(new ChatMessage.RequestBody
{
Query = "ping",
User = "test-user"
Query = "ping",
User = "test-user"
});

return new
Expand Down
263 changes: 151 additions & 112 deletions DifySharp.Test/Apis/KnowledgeBaseApiTest/DocumentApiTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,130 +1,169 @@
using System.Text.Json;
using DifySharp.KnowledgeBase;
using DifySharp.KnowledgeBase.Dataset;
using DifySharp.KnowledgeBase.Document;
using JetBrains.Annotations;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using WebApiClientCore.Parameters;

namespace DifySharp.Test.Apis.KnowledgeBaseApiTest;

public class DocumentApiTestFixture : KnowledgeBaseApiTestFixture
{
public Dataset Dataset { get; private set; }
public KnowledgeBaseClient Client { get; }

public DocumentApiTestFixture()
{
Client = ServiceProvider.GetRequiredKeyedService<KnowledgeBaseClient>("knowledge");

// create a dataset
var uuid = Guid.NewGuid().ToString("N")[..6];
Dataset =
Client.PostCreateDatasetAsync(new Create.RequestBody(Name: $"Test Dataset {uuid}"))
.GetAwaiter()
.GetResult();
}

public override void Dispose()
{
Client.DeleteDataset(Dataset.Id).GetAwaiter().GetResult();
Client.Dispose();
base.Dispose();
}
public Dataset Dataset { get; private set; }
public KnowledgeBaseClient Client { get; }

public DocumentApiTestFixture()
{
Client = ServiceProvider.GetRequiredKeyedService<KnowledgeBaseClient>("knowledge");

// create a dataset
var uuid = Guid.NewGuid().ToString("N")[..6];
Dataset =
Client.PostCreateDatasetAsync(new Create.RequestBody(Name: $"Test Dataset {uuid}"))
.GetAwaiter()
.GetResult();
}

public override void Dispose()
{
Client.DeleteDataset(Dataset.Id).GetAwaiter().GetResult();
Client.Dispose();
base.Dispose();
}
}

[TestSubject(typeof(IDocumentApi))]
public class DocumentApiTest(
DocumentApiTestFixture fixture,
ILogger<DocumentApiTest> logger
DocumentApiTestFixture fixture,
ILogger<DocumentApiTest> logger
) : IClassFixture<DocumentApiTestFixture>
{
private static Document? Document { get; set; }
private Dataset Dataset => fixture.Dataset;

private KnowledgeBaseClient Client => fixture.Client;

[Fact, TestPriority(1)]
public async Task TestCreateDocumentByText_ShouldHaveDocumentInfoInResponse()
{
Assert.Null(Document);

var uuid = Guid.NewGuid().ToString("N")[..6];

var response = await Client.PostCreateDocumentByTextAsync(
Dataset.Id,
new CreateByText.RequestBody(
$"Test Document {uuid}",
"Test Content",
IndexingTechnique.Economy,
DocForm.TextModel,
"",
new ProcessRule(
"automatic",
new Rules(
[
new PreProcessingRule(
"remove_extra_spaces",
true
),
new PreProcessingRule(
"remove_urls_emails",
true
)
],
new Segmentation(
"\n\n",
1000
),
"paragraph",
new SubChunkSegmentation(
"\n\n",
1000,
200
)
)
),
new CreateByText.RetrievalModel(
CreateByText.SearchMethod.HybridSearch,
false,
new CreateByText.RerankingModel(
"",
""
),
4,
false,
0.9f
),
"",
""
));


Assert.NotNull(response.Document);
Assert.NotEmpty(response.Document.Id);
Document = response.Document;
}

[Fact, TestPriority(2)]
public async Task TestListDocument_ShouldContainsDocumentInDataset()
{
Assert.NotNull(Document);

var response = await Client.GetDocuments(Dataset.Id);

var documents = response.Data;

Assert.Contains(documents, doc => doc.Id == Document.Id);
}

[Fact, TestPriority(3)]
public async Task TestDeleteDocument_ShouldNotContainsDocumentInDataset()
{
Assert.NotNull(Document);

var response = await Client.DeleteDocument(Dataset.Id, Document.Id);
Assert.Equal("success", response.Result);

var documents = await Client.GetDocuments(Dataset.Id);
Assert.DoesNotContain(documents.Data, doc => doc.Id == Document.Id);
}
private static List<Document> Documents { get; set; } = [];
private Dataset Dataset => fixture.Dataset;

private KnowledgeBaseClient Client => fixture.Client;

private readonly ProcessRule _defaultProcessRule = new(
"automatic",
new Rules(
[
new PreProcessingRule(
"remove_extra_spaces",
true
),
new PreProcessingRule(
"remove_urls_emails",
true
)
],
new Segmentation(
"\n\n",
1000
),
"paragraph",
new SubChunkSegmentation(
"\n\n",
1000,
200
)
)
);

private readonly CreateByText.RetrievalModel _defaultRetrievalModel = new(
CreateByText.SearchMethod.HybridSearch,
false,
new CreateByText.RerankingModel(
"",
""
),
4,
false,
0.9f
);

[Fact, TestPriority(1)]
public async Task TestCreateDocumentByText_ShouldHaveDocumentInfoInResponse()
{
// Assert.Null(Document);

var uuid = Guid.NewGuid().ToString("N")[..6];

var response = await Client.PostCreateDocumentByTextAsync(
Dataset.Id,
new CreateByText.RequestBody(
$"Test Document {uuid}",
"Test Content",
IndexingTechnique.Economy,
DocForm.TextModel,
"",
_defaultProcessRule,
_defaultRetrievalModel,
"",
""
));


Assert.NotNull(response.Document);
Assert.NotEmpty(response.Document.Id);
Documents.Add(response.Document);
}


[Fact, TestPriority(1)]
public async Task TestCreateDocumentByFile_shouldHaveDocumentInfoInResponse()
{
var uuid = Guid.NewGuid().ToString("N")[..6];

var tempFileName = $"Test Document {uuid}.md";
var tempFilePath = Path.Combine(Path.GetTempPath(), tempFileName);
await File.WriteAllTextAsync(tempFilePath, "Test Content");
await File.WriteAllTextAsync(tempFilePath, "Test Content2");
await File.WriteAllTextAsync(tempFilePath, "Test Content3");

var data = new CreateByFile.Data(
null,
IndexingTechnique.Economy,
DocForm.TextModel,
"",
_defaultProcessRule
);
var response = await Client.PostCreateDocumentByFileAsync( // tmpFilePath
Dataset.Id, data, new FormDataFile(tempFilePath));

Assert.NotNull(response.Document);
Assert.NotEmpty(response.Document.Id);
Documents.Add(response.Document);
}

[Fact, TestPriority(2)]
public async Task TestListDocument_ShouldContainsDocumentInDataset()
{
var response = await Client.GetDocuments(Dataset.Id);

var documents = response.Data;

foreach (var doc in Documents)
{
Assert.Contains(documents, d => d.Id == doc.Id);
}
}

[Fact, TestPriority(3)]
public async Task TestDeleteDocument_ShouldNotContainsDocumentInDataset()
{
foreach (var doc in Documents)
{
var response = await Client.DeleteDocument(Dataset.Id, doc.Id);
Assert.Equal("success", response.Result);
}

var documents = await Client.GetDocuments(Dataset.Id);

foreach (var doc in Documents)
{
Assert.DoesNotContain(documents.Data, d => d.Id == doc.Id);
}
}
}
Loading
Loading