Merge pull request #7660 from AppFlowy-IO/ai_writer_test

Ai writer test
This commit is contained in:
Nathan.fooo 2025-04-25 11:18:51 +08:00 committed by GitHub
commit d0b3a0dda3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 486 additions and 71 deletions

View File

@ -28,7 +28,7 @@ env:
FLUTTER_VERSION: "3.27.4"
RUST_TOOLCHAIN: "1.81.0"
CARGO_MAKE_VERSION: "0.37.18"
CLOUD_VERSION: 0.6.54-amd64
CLOUD_VERSION: 0.9.37-amd64
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

View File

@ -2,7 +2,7 @@ import 'data_migration/data_migration_test_runner.dart'
as data_migration_test_runner;
import 'database/database_test_runner.dart' as database_test_runner;
import 'document/document_test_runner.dart' as document_test_runner;
import 'set_env.dart' as preset_af_cloud_env_test;
// import 'set_env.dart' as preset_af_cloud_env_test;
import 'sidebar/sidebar_icon_test.dart' as sidebar_icon_test;
import 'sidebar/sidebar_move_page_test.dart' as sidebar_move_page_test;
import 'sidebar/sidebar_rename_untitled_test.dart'
@ -12,7 +12,7 @@ import 'uncategorized/uncategorized_test_runner.dart'
import 'workspace/workspace_test_runner.dart' as workspace_test_runner;
Future<void> main() async {
preset_af_cloud_env_test.main();
// preset_af_cloud_env_test.main();
data_migration_test_runner.main();

View File

@ -1,12 +1,19 @@
import 'package:appflowy/ai/service/ai_entities.dart';
import 'package:appflowy/env/cloud_env.dart';
import 'package:appflowy/generated/locale_keys.g.dart';
import 'package:appflowy/plugins/document/presentation/editor_plugins/ai/operations/ai_writer_entities.dart';
import 'package:appflowy/plugins/document/presentation/editor_plugins/plugins.dart';
import 'package:appflowy_backend/protobuf/flowy-ai/entities.pbenum.dart';
import 'package:appflowy_backend/protobuf/flowy-folder/view.pb.dart';
import 'package:appflowy_editor/appflowy_editor.dart';
import 'package:easy_localization/easy_localization.dart';
import 'package:flutter/services.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:integration_test/integration_test.dart';
import '../../../shared/ai_test_op.dart';
import '../../../shared/constants.dart';
import '../../../shared/mock/mock_ai.dart';
import '../../../shared/util.dart';
void main() {
@ -17,6 +24,7 @@ void main() {
(tester) async {
await tester.initializeAppFlowy(
cloudType: AuthenticatorType.appflowyCloudSelfHost,
aiRepositoryBuilder: () => MockAIRepository(),
);
await tester.tapGoogleLoginInButton();
await tester.expectToSeeHomePageWithGetStartedPage();
@ -43,5 +51,184 @@ void main() {
// expect the ai writer block is not in the document
expect(find.byType(AiWriterBlockComponent), findsNothing);
});
testWidgets('Improve writing', (tester) async {
await tester.initializeAppFlowy(
cloudType: AuthenticatorType.appflowyCloudSelfHost,
);
await tester.tapGoogleLoginInButton();
await tester.expectToSeeHomePageWithGetStartedPage();
const pageName = 'Document';
await tester.createNewPageInSpace(
spaceName: Constants.generalSpaceName,
layout: ViewLayoutPB.Document,
pageName: pageName,
);
await tester.editor.tapLineOfEditorAt(0);
// insert a paragraph
final text = 'I have an apple';
await tester.editor.tapLineOfEditorAt(0);
await tester.ime.insertText(text);
await tester.editor.updateSelection(
Selection(
start: Position(path: [0]),
end: Position(path: [0], offset: text.length),
),
);
await tester.pumpAndSettle();
await tester.tapButton(find.byType(ImproveWritingButton));
final editorState = tester.editor.getCurrentEditorState();
final document = editorState.document;
expect(document.root.children.length, 3);
expect(document.root.children[1].type, ParagraphBlockKeys.type);
expect(
document.root.children[1].delta!.toPlainText(),
'I have an apple and a banana',
);
});
testWidgets('fix grammar', (tester) async {
await tester.initializeAppFlowy(
cloudType: AuthenticatorType.appflowyCloudSelfHost,
);
await tester.tapGoogleLoginInButton();
await tester.expectToSeeHomePageWithGetStartedPage();
const pageName = 'Document';
await tester.createNewPageInSpace(
spaceName: Constants.generalSpaceName,
layout: ViewLayoutPB.Document,
pageName: pageName,
);
await tester.editor.tapLineOfEditorAt(0);
// insert a paragraph
final text = 'We didnt had enough money';
await tester.editor.tapLineOfEditorAt(0);
await tester.ime.insertText(text);
await tester.editor.updateSelection(
Selection(
start: Position(path: [0]),
end: Position(path: [0], offset: text.length),
),
);
await tester.pumpAndSettle();
await tester.selectAIWriter(AiWriterCommand.fixSpellingAndGrammar);
final editorState = tester.editor.getCurrentEditorState();
final document = editorState.document;
expect(document.root.children.length, 3);
expect(document.root.children[1].type, ParagraphBlockKeys.type);
expect(
document.root.children[1].delta!.toPlainText(),
'We didnt have enough money',
);
});
testWidgets('ask ai', (tester) async {
await tester.initializeAppFlowy(
cloudType: AuthenticatorType.appflowyCloudDevelop,
aiRepositoryBuilder: () => MockAIRepository(
validator: _CompletionHistoryValidator(),
),
);
await tester.tapGoogleLoginInButton();
await tester.expectToSeeHomePageWithGetStartedPage();
const pageName = 'Document';
await tester.createNewPageInSpace(
spaceName: Constants.generalSpaceName,
layout: ViewLayoutPB.Document,
pageName: pageName,
);
await tester.editor.tapLineOfEditorAt(0);
// insert a paragraph
final text = 'What is TPU?';
await tester.editor.tapLineOfEditorAt(0);
await tester.ime.insertText(text);
await tester.editor.updateSelection(
Selection(
start: Position(path: [0]),
end: Position(path: [0], offset: text.length),
),
);
await tester.pumpAndSettle();
await tester.selectAIWriter(AiWriterCommand.userQuestion);
await tester.enterTextInPromptTextField("Explain the concept of TPU");
await tester.simulateKeyEvent(LogicalKeyboardKey.enter);
await tester.pumpAndSettle();
// await tester.selectModel("GPT-4o-mini");
await tester.enterTextInPromptTextField("How about GPU?");
await tester.simulateKeyEvent(LogicalKeyboardKey.enter);
await tester.pumpAndSettle();
});
});
}
class _CompletionHistoryValidator extends StreamCompletionValidator {
static const _expectedResponses = {
1: "What is TPU?",
3: [
"What is TPU?",
"Explain the concept of TPU",
"TPU is a tensor processing unit that is designed to accelerate machine",
],
5: [
"What is TPU?",
"Explain the concept of TPU",
"TPU is a tensor processing unit that is designed to accelerate machine",
"How about GPU?",
"GPU is a graphics processing unit that is designed to accelerate machine learning tasks.",
],
};
@override
bool validate(
String text,
String? objectId,
CompletionTypePB completionType,
PredefinedFormat? format,
List<String> sourceIds,
List<AiWriterRecord> history,
) {
assert(completionType == CompletionTypePB.UserQuestion);
if (history.isEmpty) return false;
final expectedMessages = _expectedResponses[history.length];
if (expectedMessages == null) return false;
if (expectedMessages is String) {
_assertMessage(history[0].content, expectedMessages);
return true;
} else if (expectedMessages is List<String>) {
for (var i = 0; i < expectedMessages.length; i++) {
_assertMessage(history[i].content, expectedMessages[i]);
}
return true;
}
return false;
}
void _assertMessage(String actual, String expected) {
assert(
actual.trim() == expected,
"expected '$expected', but got '${actual.trim()}'",
);
}
}

View File

@ -8,11 +8,13 @@ import '../../shared/util.dart';
void main() {
IntegrationTestWidgetsFlutterBinding.ensureInitialized();
group('Empty', () {
testWidgets('set appflowy cloud', (tester) async {
group('Preset cloud env', () {
testWidgets('use self-hosted cloud', (tester) async {
await tester.initializeAppFlowy(
cloudType: AuthenticatorType.appflowyCloudSelfHost,
);
await tester.pumpAndSettle();
});
});
}

View File

@ -0,0 +1,41 @@
import 'package:appflowy/ai/ai.dart';
import 'package:appflowy/plugins/document/presentation/editor_plugins/ai/ai_writer_toolbar_item.dart';
import 'package:appflowy/plugins/document/presentation/editor_plugins/ai/operations/ai_writer_entities.dart';
import 'package:flutter/material.dart';
import 'package:flutter_test/flutter_test.dart';
import 'util.dart';
extension AppFlowyAITest on WidgetTester {
Future<void> selectAIWriter(AiWriterCommand command) async {
await tapButton(find.byType(AiWriterToolbarActionList));
await tapButton(find.text(command.i18n));
await pumpAndSettle();
}
Future<void> selectModel(String modelName) async {
await tapButton(find.byType(SelectModelMenu));
await tapButton(find.text(modelName));
await pumpAndSettle();
}
Future<void> enterTextInPromptTextField(String text) async {
// Wait for the text field to be visible
await pumpAndSettle();
// Find the ExtendedTextField widget
final textField = find.descendant(
of: find.byType(PromptInputTextField),
matching: find.byType(TextField),
);
expect(textField, findsOneWidget, reason: 'ExtendedTextField not found');
final widget = element(textField).widget as TextField;
expect(widget.enabled, isTrue, reason: 'TextField is not enabled');
await tap(textField);
testTextInput.enterText(text);
await pumpAndSettle(const Duration(milliseconds: 300));
}
}

View File

@ -1,6 +1,7 @@
import 'dart:async';
import 'dart:io';
import 'package:appflowy/ai/service/appflowy_ai_service.dart';
import 'package:appflowy/env/cloud_env.dart';
import 'package:appflowy/env/cloud_env_test.dart';
import 'package:appflowy/startup/entry_point.dart';
@ -20,6 +21,8 @@ import 'package:path/path.dart' as p;
import 'package:path_provider/path_provider.dart';
import 'package:universal_platform/universal_platform.dart';
import 'mock/mock_ai.dart';
class FlowyTestContext {
FlowyTestContext({required this.applicationDataDirectory});
@ -33,13 +36,14 @@ extension AppFlowyTestBase on WidgetTester {
// use to specify the application data directory, if not specified, a temporary directory will be used.
String? dataDirectory,
Size windowSize = const Size(1600, 1200),
AuthenticatorType? cloudType,
String? email,
AuthenticatorType? cloudType,
AIRepository Function()? aiRepositoryBuilder,
}) async {
if (Platform.isLinux || Platform.isWindows || Platform.isMacOS) {
// Set the window size
await binding.setSurfaceSize(windowSize);
}
//cloudType = AuthenticatorType.appflowyCloudDevelop;
mockHotKeyManagerHandlers();
final applicationDataDirectory = dataDirectory ??
@ -50,48 +54,87 @@ extension AppFlowyTestBase on WidgetTester {
await FlowyRunner.run(
AppFlowyApplication(),
IntegrationMode.integrationTest,
rustEnvsBuilder: () {
rustEnvsBuilder: () => _buildRustEnvs(cloudType),
didInitGetItCallback: () => _initializeCloudServices(
cloudType: cloudType,
email: email,
aiRepositoryBuilder: aiRepositoryBuilder,
),
);
await waitUntilSignInPageShow();
return FlowyTestContext(
applicationDataDirectory: applicationDataDirectory,
);
}
Map<String, String> _buildRustEnvs(AuthenticatorType? cloudType) {
final rustEnvs = <String, String>{};
if (cloudType != null) {
switch (cloudType) {
case AuthenticatorType.local:
break;
case AuthenticatorType.appflowyCloudSelfHost:
case AuthenticatorType.appflowyCloudDevelop:
rustEnvs["GOTRUE_ADMIN_EMAIL"] = "admin@example.com";
rustEnvs["GOTRUE_ADMIN_PASSWORD"] = "password";
break;
default:
throw Exception("not supported");
throw Exception("Unsupported cloud type: $cloudType");
}
}
return rustEnvs;
},
didInitGetItCallback: () {
return Future(
() async {
if (cloudType != null) {
}
Future<void> _initializeCloudServices({
required AuthenticatorType? cloudType,
String? email,
AIRepository Function()? aiRepositoryBuilder,
}) async {
if (cloudType == null) return;
switch (cloudType) {
case AuthenticatorType.local:
await useLocalServer();
break;
case AuthenticatorType.appflowyCloudSelfHost:
await useTestSelfHostedAppFlowyCloud();
await _setupAppFlowyCloud(
useLocal: false,
email: email,
aiRepositoryBuilder: aiRepositoryBuilder,
);
break;
case AuthenticatorType.appflowyCloudDevelop:
await _setupAppFlowyCloud(
useLocal: integrationMode().isDevelop,
email: email,
aiRepositoryBuilder: aiRepositoryBuilder,
);
break;
default:
throw Exception("Unsupported cloud type: $cloudType");
}
}
Future<void> _setupAppFlowyCloud({
required bool useLocal,
String? email,
AIRepository Function()? aiRepositoryBuilder,
}) async {
if (useLocal) {
await useAppFlowyCloudDevelop("http://localhost");
} else {
await useSelfHostedAppFlowyCloud(TestEnv.afCloudUrl);
}
getIt.unregister<AuthService>();
getIt.unregister<AIRepository>();
getIt.registerFactory<AuthService>(
() => AppFlowyCloudMockAuthService(email: email),
);
default:
throw Exception("not supported");
}
}
},
);
},
);
await waitUntilSignInPageShow();
return FlowyTestContext(
applicationDataDirectory: applicationDataDirectory,
getIt.registerFactory<AIRepository>(
aiRepositoryBuilder ?? () => MockAIRepository(),
);
}
@ -275,10 +318,6 @@ extension AppFlowyFinderTestBase on CommonFinders {
}
}
Future<void> useTestSelfHostedAppFlowyCloud() async {
await useSelfHostedAppFlowyCloudWithURL(TestEnv.afCloudUrl);
}
Future<String> mockApplicationDataStorage({
// use to append after the application data directory
String? pathExtension,

View File

@ -0,0 +1,128 @@
import 'dart:async';
import 'package:appflowy/ai/service/ai_entities.dart';
import 'package:appflowy/ai/service/appflowy_ai_service.dart';
import 'package:appflowy/ai/service/error.dart';
import 'package:appflowy/plugins/document/presentation/editor_plugins/ai/operations/ai_writer_entities.dart';
import 'package:appflowy_backend/protobuf/flowy-ai/entities.pbenum.dart';
import 'package:mocktail/mocktail.dart';
final _mockAiMap = <CompletionTypePB, Map<String, List<String>>>{
CompletionTypePB.ImproveWriting: {
"I have an apple": [
"I",
"have",
"an",
"apple",
"and",
"a",
"banana",
],
},
CompletionTypePB.SpellingAndGrammar: {
"We didnt had enough money": [
"We",
"didnt",
"have",
"enough",
"money",
],
},
CompletionTypePB.UserQuestion: {
"Explain the concept of TPU": [
"TPU",
"is",
"a",
"tensor",
"processing",
"unit",
"that",
"is",
"designed",
"to",
"accelerate",
"machine",
],
"How about GPU?": [
"GPU",
"is",
"a",
"graphics",
"processing",
"unit",
"that",
"is",
"designed",
"to",
"accelerate",
"machine",
"learning",
"tasks",
],
},
};
abstract class StreamCompletionValidator {
bool validate(
String text,
String? objectId,
CompletionTypePB completionType,
PredefinedFormat? format,
List<String> sourceIds,
List<AiWriterRecord> history,
);
}
class MockCompletionStream extends Mock implements CompletionStream {}
class MockAIRepository extends Mock implements AppFlowyAIService {
MockAIRepository({this.validator});
StreamCompletionValidator? validator;
@override
Future<(String, CompletionStream)?> streamCompletion({
String? objectId,
required String text,
PredefinedFormat? format,
List<String> sourceIds = const [],
List<AiWriterRecord> history = const [],
required CompletionTypePB completionType,
required Future<void> Function() onStart,
required Future<void> Function(String text) processMessage,
required Future<void> Function(String text) processAssistMessage,
required Future<void> Function() onEnd,
required void Function(AIError error) onError,
required void Function(LocalAIStreamingState state)
onLocalAIStreamingStateChange,
}) async {
if (validator != null) {
if (!validator!.validate(
text,
objectId,
completionType,
format,
sourceIds,
history,
)) {
throw Exception('Invalid completion');
}
}
final stream = MockCompletionStream();
unawaited(
Future(() async {
await onStart();
final lines = _mockAiMap[completionType]?[text.trim()];
if (lines == null) {
throw Exception('No mock ai found for $text and $completionType');
}
for (final line in lines) {
await processMessage('$line ');
}
await onEnd();
}),
);
return ('mock_id', stream);
}
}

View File

@ -23,7 +23,7 @@ enum LocalAIStreamingState {
}
abstract class AIRepository {
Future<void> streamCompletion({
Future<(String, CompletionStream)?> streamCompletion({
String? objectId,
required String text,
PredefinedFormat? format,

View File

@ -167,11 +167,16 @@ Future<void> useBaseWebDomain(String? url) async {
);
}
Future<void> useSelfHostedAppFlowyCloudWithURL(String url) async {
Future<void> useSelfHostedAppFlowyCloud(String url) async {
await _setAuthenticatorType(AuthenticatorType.appflowyCloudSelfHost);
await _setAppFlowyCloudUrl(url);
}
Future<void> useAppFlowyCloudDevelop(String url) async {
await _setAuthenticatorType(AuthenticatorType.appflowyCloudDevelop);
await _setAppFlowyCloudUrl(url);
}
Future<void> useAppFlowyBetaCloudWithURL(
String url,
AuthenticatorType authenticatorType,

View File

@ -87,7 +87,7 @@ class _SelfHostUrlBottomSheetState extends State<SelfHostUrlBottomSheet> {
case SelfHostUrlBottomSheetType.shareDomain:
await useBaseWebDomain(url);
case SelfHostUrlBottomSheetType.cloudURL:
await useSelfHostedAppFlowyCloudWithURL(url);
await useSelfHostedAppFlowyCloud(url);
}
await runAppFlowy();
},

View File

@ -1,6 +1,7 @@
import 'dart:async';
import 'package:appflowy/ai/ai.dart';
import 'package:appflowy/startup/startup.dart';
import 'package:appflowy/workspace/application/view/view_service.dart';
import 'package:appflowy_backend/dispatch/dispatch.dart';
import 'package:appflowy_backend/log.dart';
@ -28,15 +29,14 @@ class AiWriterCubit extends Cubit<AiWriterState> {
this.onCreateNode,
this.onRemoveNode,
this.onAppendToDocument,
AppFlowyAIService? aiService,
}) : _aiService = aiService ?? AppFlowyAIService(),
}) : _aiService = getIt<AIRepository>(),
_textRobot = MarkdownTextRobot(editorState: editorState),
selectedSourcesNotifier = ValueNotifier([documentId]),
super(IdleAiWriterState());
final String documentId;
final EditorState editorState;
final AppFlowyAIService _aiService;
final AIRepository _aiService;
final MarkdownTextRobot _textRobot;
final void Function()? onCreateNode;
final void Function()? onRemoveNode;
@ -306,12 +306,14 @@ class AiWriterCubit extends Cubit<AiWriterState> {
// check the node is registered
if (node == null) {
Log.warn('[AI writer] Node is null');
return (false, '');
}
// check the selection is valid
final selection = node.aiWriterSelection?.normalized;
if (selection == null) {
Log.warn('[AI writer]Selection is null');
return (false, '');
}

View File

@ -58,6 +58,7 @@ class _AiWriterScrollWrapperState extends State<AiWriterScrollWrapper> {
@override
void dispose() {
aiWriterCubit.close();
throttler.dispose();
super.dispose();
}

View File

@ -1,3 +1,4 @@
import 'package:appflowy/ai/service/appflowy_ai_service.dart';
import 'package:appflowy/core/config/kv.dart';
import 'package:appflowy/core/network_monitor.dart';
import 'package:appflowy/env/cloud_env.dart';
@ -60,6 +61,7 @@ Future<void> _resolveCloudDeps(GetIt getIt) async {
final env = await AppFlowyCloudSharedEnv.fromEnv();
Log.info("cloud setting: $env");
getIt.registerFactory<AppFlowyCloudSharedEnv>(() => env);
getIt.registerFactory<AIRepository>(() => AppFlowyAIService());
if (isAppFlowyCloudEnabled) {
getIt.registerSingleton(

View File

@ -48,7 +48,7 @@ class AppFlowyCloudURLsBloc
await validateUrl(state.updatedServerUrl).fold(
(url) async {
await useSelfHostedAppFlowyCloudWithURL(url);
await useSelfHostedAppFlowyCloud(url);
isSuccess = true;
},
(err) async => emit(state.copyWith(urlError: err)),

View File

@ -4,6 +4,7 @@ import 'package:appflowy/ai/ai.dart';
import 'package:appflowy/plugins/document/presentation/editor_plugins/ai/operations/ai_writer_cubit.dart';
import 'package:appflowy/plugins/document/presentation/editor_plugins/ai/operations/ai_writer_entities.dart';
import 'package:appflowy/plugins/document/presentation/editor_plugins/plugins.dart';
import 'package:appflowy/startup/startup.dart';
import 'package:appflowy_backend/protobuf/flowy-ai/entities.pb.dart';
import 'package:appflowy_editor/appflowy_editor.dart';
import 'package:bloc_test/bloc_test.dart';
@ -145,6 +146,13 @@ class _MockErrorRepository extends Mock implements AppFlowyAIService {
}
}
void registerMockRepository(AppFlowyAIService mock) {
if (getIt.isRegistered<AIRepository>()) {
getIt.unregister<AIRepository>();
}
getIt.registerFactory<AIRepository>(() => mock);
}
void main() {
group('AIWriterCubit:', () {
const text1 = '1. Select text to style using the toolbar menu.';
@ -174,10 +182,10 @@ void main() {
);
final editorState = EditorState(document: document)
..selection = selection;
registerMockRepository(_MockAIRepository());
return AiWriterCubit(
documentId: '',
editorState: editorState,
aiService: _MockAIRepository(),
);
},
act: (bloc) => bloc.register(
@ -230,10 +238,10 @@ void main() {
);
final editorState = EditorState(document: document)
..selection = selection;
registerMockRepository(_MockErrorRepository());
return AiWriterCubit(
documentId: '',
editorState: editorState,
aiService: _MockErrorRepository(),
);
},
act: (bloc) => bloc.register(
@ -279,10 +287,10 @@ void main() {
final editorState = EditorState(document: document)
..selection = selection;
final aiNode = editorState.getNodeAtPath([3])!;
registerMockRepository(_MockAIRepository());
final bloc = AiWriterCubit(
documentId: '',
editorState: editorState,
aiService: _MockAIRepository(),
);
bloc.register(aiNode);
await blocResponseFuture();
@ -327,10 +335,10 @@ void main() {
final editorState = EditorState(document: document)
..selection = selection;
final aiNode = editorState.getNodeAtPath([3])!;
registerMockRepository(_MockAIRepository());
final bloc = AiWriterCubit(
documentId: '',
editorState: editorState,
aiService: _MockAIRepository(),
);
bloc.register(aiNode);
await blocResponseFuture();
@ -366,10 +374,10 @@ void main() {
final editorState = EditorState(document: document)
..selection = selection;
final aiNode = editorState.getNodeAtPath([3])!;
registerMockRepository(_MockAIRepositoryLess());
final bloc = AiWriterCubit(
documentId: '',
editorState: editorState,
aiService: _MockAIRepositoryLess(),
);
bloc.register(aiNode);
await blocResponseFuture();
@ -403,10 +411,10 @@ void main() {
final editorState = EditorState(document: document)
..selection = selection;
final aiNode = editorState.getNodeAtPath([3])!;
registerMockRepository(_MockAIRepositoryMore());
final bloc = AiWriterCubit(
documentId: '',
editorState: editorState,
aiService: _MockAIRepositoryMore(),
);
bloc.register(aiNode);
await blocResponseFuture();