diff --git a/appinfo/info.xml b/appinfo/info.xml index e87e2fb3..997ce70c 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -62,7 +62,7 @@ Known providers: More details on how to set this up in the [admin docs](https://docs.nextcloud.com/server/latest/admin_manual/ai/index.html) ]]> - 4.1.0 + 4.2.1 agpl Julien Veyssier Assistant diff --git a/appinfo/routes.php b/appinfo/routes.php index 1dc8ada3..f05919df 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -43,6 +43,11 @@ ['name' => 'agentSkillsApi#storeSkill', 'url' => '/api/{apiVersion}/skills', 'verb' => 'POST', 'requirements' => $requirements], ['name' => 'agentSkillsApi#loadSkill', 'url' => '/api/{apiVersion}/skills/{skillName}', 'verb' => 'GET', 'requirements' => $requirements], + ['name' => 'agentsApi#listAgents', 'url' => '/api/{apiVersion}/agents', 'verb' => 'GET', 'requirements' => $requirements], + ['name' => 'agentsApi#createAgent', 'url' => '/api/{apiVersion}/agents', 'verb' => 'POST', 'requirements' => $requirements], + ['name' => 'agentsApi#updateAgent', 'url' => '/api/{apiVersion}/agents/{id}', 'verb' => 'PUT', 'requirements' => $requirements], + ['name' => 'agentsApi#deleteAgent', 'url' => '/api/{apiVersion}/agents/{id}', 'verb' => 'DELETE', 'requirements' => $requirements], + ['name' => 'chattyLLM#newSession', 'url' => '/chat/sessions', 'verb' => 'POST', 'postfix' => 'restful'], ['name' => 'chattyLLM#updateChatSession', 'url' => '/chat/sessions/{sessionId}', 'verb' => 'PUT', 'postfix' => 'restful'], ['name' => 'chattyLLM#deleteSession', 'url' => '/chat/sessions/{sessionId}', 'verb' => 'DELETE', 'postfix' => 'restful'], diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 332774e4..fc589f94 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -18,6 +18,8 @@ use OCA\Assistant\Listener\NewFileMenuTaskFailedListener; use OCA\Assistant\Listener\NewFileMenuTaskSuccessfulListener; use OCA\Assistant\Listener\SpeechToText\SpeechToTextReferenceListener; +use OCA\Assistant\Listener\Talk\AgentTalkMessageListener; +use OCA\Assistant\Listener\Talk\AgentReactionListener; use OCA\Assistant\Listener\TaskFailedListener; use OCA\Assistant\Listener\TaskOutputFileReferenceListener; use OCA\Assistant\Listener\TaskSuccessfulListener; @@ -110,6 +112,16 @@ public function register(IRegistrationContext $context): void { $context->registerEventListener(UserDeletedEvent::class, UserDeletedListener::class); + // the Talk event classes are not necessarily autoloadable when this app + // file is registered, the listeners themselves guard for Talk being absent + /** @psalm-suppress UndefinedClass, InvalidArgument */ + $context->registerEventListener(\OCA\Talk\Events\ChatMessageSentEvent::class, AgentTalkMessageListener::class); + /** @psalm-suppress UndefinedClass, InvalidArgument */ + $context->registerEventListener(\OCA\Talk\Events\ReactionAddedEvent::class, AgentReactionListener::class); + // file shares (voice messages, attachments) are "file_shared" system messages + /** @psalm-suppress UndefinedClass, InvalidArgument */ + $context->registerEventListener(\OCA\Talk\Events\SystemMessageSentEvent::class, AgentTalkMessageListener::class); + if (class_exists('OCP\\TaskProcessing\\TaskTypes\\AudioToAudioChat')) { $context->registerTaskProcessingProvider(AudioToAudioChatProvider::class); } diff --git a/lib/Controller/AgentsApiController.php b/lib/Controller/AgentsApiController.php new file mode 100644 index 00000000..1e802e04 --- /dev/null +++ b/lib/Controller/AgentsApiController.php @@ -0,0 +1,232 @@ +, access_groups: list}> + */ + private function serializeAgents(): array { + $agents = $this->agentsService->getAllAgents(); + $result = []; + foreach ($agents as $agent) { + $result[] = $this->serializeAgent($agent); + } + return $result; + } + + private function serializeAgent(Agent $agent): array { + $user = $this->userManager->get($agent->getUserId()); + $access = $this->agentsService->getAgentAccess((int)$agent->getId()); + return [ + 'id' => (int)$agent->getId(), + 'user_id' => $agent->getUserId(), + 'display_name' => $user?->getDisplayName() ?: $agent->getUserId(), + 'personality' => $agent->getPersonality(), + 'scope' => $agent->getScope(), + 'access_users' => $access['users'], + 'access_groups' => $access['groups'], + ]; + } + + private function checkAdmin(): bool { + $user = $this->userSession->getUser(); + if ($user === null) { + return false; + } + return $this->groupManager->isInGroup($user->getUID(), 'admin'); + } + + /** + * @param list $accessUsers + * @param list $accessGroups + * @return string|null A localized error message, or null when all entries are valid + */ + private function validateAccessLists(array $accessUsers, array $accessGroups): ?string { + foreach ($accessUsers as $accessUserId) { + if (!is_string($accessUserId) || $this->userManager->get($accessUserId) === null) { + return $this->l10n->t('The user "%s" does not exist', [$accessUserId]); + } + } + foreach ($accessGroups as $accessGroupId) { + if (!is_string($accessGroupId) || $this->groupManager->get($accessGroupId) === null) { + return $this->l10n->t('The group "%s" does not exist', [$accessGroupId]); + } + } + return null; + } + + /** + * List all agents (admin only) + * + * @return DataResponse, access_groups: list}>}, array{}>|DataResponse + * + * 200: Agents returned + * 403: User is not an admin + */ + #[OpenAPI(scope: OpenAPI::SCOPE_DEFAULT, tags: ['agents'])] + public function listAgents(): DataResponse { + if (!$this->checkAdmin()) { + return new DataResponse(['error' => $this->l10n->t('Only administrators can manage agents')], Http::STATUS_FORBIDDEN); + } + return new DataResponse(['agents' => $this->serializeAgents()]); + } + + /** + * Create an agent (admin only) + * + * A dedicated Nextcloud user is created in the default database backend if it does not exist yet. + * + * @param string $name The name of the agent + * @param string $userId The user id of the agent's dedicated Nextcloud user + * @param string $personality The agent's system prompt + * @param string $scope Optional scope for later use + * @param list $accessUsers User ids that may access the agent (empty = every user) + * @param list $accessGroups Group ids that may access the agent (empty = every user) + * @return DataResponse, access_groups: list}}, array{}>|DataResponse|DataResponse|DataResponse + * + * 201: Agent created + * 400: Invalid data + * 403: User is not an admin + * 500: Agent creation failed + */ + #[OpenAPI(scope: OpenAPI::SCOPE_DEFAULT, tags: ['agents'])] + public function createAgent( + string $name = '', string $userId = '', string $personality = '', string $scope = '', + array $accessUsers = [], array $accessGroups = [], + ): DataResponse { + if (!$this->checkAdmin()) { + return new DataResponse(['error' => $this->l10n->t('Only administrators can manage agents')], Http::STATUS_FORBIDDEN); + } + $name = trim($name); + $userId = trim($userId); + if ($name === '' || $userId === '') { + return new DataResponse(['error' => $this->l10n->t('Name and user id are required')], Http::STATUS_BAD_REQUEST); + } + if (strlen($personality) > 64_000) { + return new DataResponse(['error' => $this->l10n->t('The agent personality is too long')], Http::STATUS_BAD_REQUEST); + } + $validationError = $this->validateAccessLists($accessUsers, $accessGroups); + if ($validationError !== null) { + return new DataResponse(['error' => $validationError], Http::STATUS_BAD_REQUEST); + } + try { + $agent = $this->agentsService->createAgent($name, $userId, $personality, $scope, $accessUsers, $accessGroups); + return new DataResponse(['agent' => $this->serializeAgent($agent)], Http::STATUS_CREATED); + } catch (AgentUserException $e) { + return new DataResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST); + } catch (\Exception|Throwable $e) { + return new DataResponse(['error' => $this->l10n->t('Failed to create the agent')], Http::STATUS_INTERNAL_SERVER_ERROR); + } + } + + /** + * Update an agent (admin only) + * + * @param int $id The agent id + * @param string|null $personality The new personality (system prompt) + * @param string|null $scope The new scope + * @param list|null $accessUsers When not null, replaces the user ids that may access the agent + * @param list|null $accessGroups When not null, replaces the group ids that may access the agent + * @return DataResponse, access_groups: list}}, array{}>|DataResponse|DataResponse|DataResponse|DataResponse + * + * 200: Agent updated + * 400: Invalid data + * 403: User is not an admin + * 404: Agent not found + * 500: Agent update failed + */ + #[OpenAPI(scope: OpenAPI::SCOPE_DEFAULT, tags: ['agents'])] + public function updateAgent( + int $id, ?string $personality = null, ?string $scope = null, + ?array $accessUsers = null, ?array $accessGroups = null, + ): DataResponse { + if (!$this->checkAdmin()) { + return new DataResponse(['error' => $this->l10n->t('Only administrators can manage agents')], Http::STATUS_FORBIDDEN); + } + $agent = $this->agentsService->getAgentById($id); + if ($agent === null) { + return new DataResponse(['error' => $this->l10n->t('Agent not found')], Http::STATUS_NOT_FOUND); + } + if ($accessUsers !== null || $accessGroups !== null) { + $validationError = $this->validateAccessLists($accessUsers ?? [], $accessGroups ?? []); + if ($validationError !== null) { + return new DataResponse(['error' => $validationError], Http::STATUS_BAD_REQUEST); + } + } + try { + $agent = $this->agentsService->updateAgent($id, $personality, $scope, $accessUsers, $accessGroups); + return new DataResponse(['agent' => $this->serializeAgent($agent)]); + } catch (AgentUserException $e) { + return new DataResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST); + } catch (\Exception|Throwable $e) { + return new DataResponse(['error' => $this->l10n->t('Failed to update the agent')], Http::STATUS_INTERNAL_SERVER_ERROR); + } + } + + /** + * Delete an agent (admin only) + * + * @param int $id The agent id + * @param int $deleteUser Whether the related Nextcloud user should be deleted too + * @return DataResponse|DataResponse|DataResponse|DataResponse + * + * 200: Agent deleted + * 403: User is not an admin + * 404: Agent not found + * 500: Agent deletion failed + */ + #[OpenAPI(scope: OpenAPI::SCOPE_DEFAULT, tags: ['agents'])] + public function deleteAgent(int $id, int $deleteUser = 0): DataResponse { + if (!$this->checkAdmin()) { + return new DataResponse(['error' => $this->l10n->t('Only administrators can manage agents')], Http::STATUS_FORBIDDEN); + } + $agent = $this->agentsService->getAgentById($id); + if ($agent === null) { + return new DataResponse(['error' => $this->l10n->t('Agent not found')], Http::STATUS_NOT_FOUND); + } + try { + $this->agentsService->deleteAgent($id, $deleteUser === 1); + return new DataResponse(['action' => 'deleted']); + } catch (\Exception|Throwable $e) { + return new DataResponse(['error' => $this->l10n->t('Failed to delete the agent')], Http::STATUS_INTERNAL_SERVER_ERROR); + } + } +} diff --git a/lib/Controller/ChattyLLMController.php b/lib/Controller/ChattyLLMController.php index 3e77102d..4bc7587a 100644 --- a/lib/Controller/ChattyLLMController.php +++ b/lib/Controller/ChattyLLMController.php @@ -12,6 +12,7 @@ use OCA\Assistant\Db\ChattyLLM\MessageMapper; use OCA\Assistant\Db\ChattyLLM\SessionMapper; use OCA\Assistant\ResponseDefinitions; +use OCA\Assistant\Service\AgencyActionService; use OCA\Assistant\Service\AgentSkillsService; use OCA\Assistant\Service\AssistantService; use OCA\Assistant\Service\BadRequestException; @@ -45,7 +46,7 @@ class ChattyLLMController extends OCSController { // must match MEMORIES_RELATIVE_FOLDER_PATH in context_agent private const MEMORIES_FOLDER_PATH = 'Context Agent/Memories'; - private array $agencyActionData; + private AgencyActionService $agencyActionService; public function __construct( string $appName, @@ -61,140 +62,22 @@ public function __construct( private ChatService $chatService, private AssistantService $assistantService, private AgentSkillsService $agentSkillsService, + AgencyActionService $agencyActionService, ) { parent::__construct($appName, $request); - $this->agencyActionData = [ - // talk - 'send_message_to_conversation' => [ - 'title' => $this->l10n->t('Send a message to a Talk conversation'), - 'icon' => 'Send', - ], - 'create_public_conversation' => [ - 'title' => $this->l10n->t('Create a conversation'), - 'icon' => 'ChatPlus', - ], - // mail - 'send_email' => [ - 'title' => $this->l10n->t('Send an email'), - 'icon' => 'EmailPlus', - ], - // calendar - 'schedule_event' => [ - 'title' => $this->l10n->t('Schedule a calendar event'), - 'icon' => 'CalendarPlus', - ], - 'add_task' => [ - 'title' => $this->l10n->t('Add a calendar task'), - 'icon' => 'CalendarCheck', - ], - // deck - 'add_card' => [ - 'title' => $this->l10n->t('Create a Deck card'), - 'icon' => 'CardPlus', - ], - 'upload_file' => [ - 'title' => $this->l10n->t('Upload file'), - 'icon' => 'Upload', - ], - 'create_folder' => [ - 'title' => $this->l10n->t('Create folder'), - 'icon' => 'FolderPlus', - ], - 'move_file' => [ - 'title' => $this->l10n->t('Move file'), - 'icon' => 'FileMove', - ], - 'copy_file' => [ - 'title' => $this->l10n->t('Copy file'), - 'icon' => 'FileCopy', - ], - 'delete_file' => [ - 'title' => $this->l10n->t('Delete file'), - 'icon' => 'Delete', - ], - 'create_bookmark' => [ - 'title' => $this->l10n->t('Create bookmark'), - 'icon' => 'BookmarkPlus', - ], - 'update_bookmark' => [ - 'title' => $this->l10n->t('Update bookmark'), - 'icon' => 'BookmarkEdit', - ], - 'delete_bookmark' => [ - 'title' => $this->l10n->t('Delete bookmark'), - 'icon' => 'BookmarkRemove', - ], - 'create_bookmark_folder' => [ - 'title' => $this->l10n->t('Create bookmark folder'), - 'icon' => 'FolderPlus', - ], - 'create_circle' => [ - 'title' => $this->l10n->t('Create team'), - 'icon' => 'AccountGroupPlus', - ], - 'add_member_to_circle' => [ - 'title' => $this->l10n->t('Add member to team'), - 'icon' => 'AccountPlus', - ], - 'remove_member_from_circle' => [ - 'title' => $this->l10n->t('Remove member from team'), - 'icon' => 'AccountRemove', - ], - 'update_circle' => [ - 'title' => $this->l10n->t('Update team'), - 'icon' => 'AccountGroupEdit', - ], - 'delete_circle' => [ - 'title' => $this->l10n->t('Delete team'), - 'icon' => 'AccountGroupRemove', - ], - 'share_with_circle' => [ - 'title' => $this->l10n->t('Share with team'), - 'icon' => 'ShareVariant', - ], - 'create_form' => [ - 'title' => $this->l10n->t('Create form'), - 'icon' => 'FormTextbox', - ], - 'add_question_to_form' => [ - 'title' => $this->l10n->t('Add question to form'), - 'icon' => 'CommentPlus', - ], - 'delete_form' => [ - 'title' => $this->l10n->t('Delete form'), - 'icon' => 'Delete', - ], - 'update_form_settings' => [ - 'title' => $this->l10n->t('Update form settings'), - 'icon' => 'Cog', - ], - 'share_with_user' => [ - 'title' => $this->l10n->t('Share with user'), - 'icon' => 'AccountShare', - ], - 'share_with_group' => [ - 'title' => $this->l10n->t('Share with group'), - 'icon' => 'AccountGroup', - ], - 'update_share_permissions' => [ - 'title' => $this->l10n->t('Update share permissions'), - 'icon' => 'LockOpen', - ], - 'delete_share' => [ - 'title' => $this->l10n->t('Delete share'), - 'icon' => 'Delete', - ], - ]; + $this->agencyActionService = $agencyActionService; } private function improveAgencyActionNames(array $actions): array { return array_map(function ($action) { - if (isset($action->name, $this->agencyActionData[$action->name])) { - if (isset($this->agencyActionData[$action->name]['icon'])) { - $action->icon = $this->agencyActionData[$action->name]['icon']; + if (isset($action->name)) { + $icon = $this->agencyActionService->getActionIcon($action->name); + $title = $this->agencyActionService->getActionTitle($action->name); + if ($icon !== null) { + $action->icon = $icon; } - if (isset($this->agencyActionData[$action->name]['title'])) { - $action->name = $this->agencyActionData[$action->name]['title']; + if ($title !== null) { + $action->name = $title; } } return $action; diff --git a/lib/Db/Agent.php b/lib/Db/Agent.php new file mode 100644 index 00000000..d8fd83e5 --- /dev/null +++ b/lib/Db/Agent.php @@ -0,0 +1,58 @@ +addType('userId', Types::STRING); + $this->addType('personality', Types::TEXT); + $this->addType('scope', Types::TEXT); + } + + #[\ReturnTypeWillChange] + public function jsonSerialize() { + return [ + 'id' => $this->getId(), + 'user_id' => $this->getUserId(), + 'personality' => $this->getPersonality(), + 'scope' => $this->getScope(), + ]; + } +} diff --git a/lib/Db/AgentMapper.php b/lib/Db/AgentMapper.php new file mode 100644 index 00000000..dd7f8c7b --- /dev/null +++ b/lib/Db/AgentMapper.php @@ -0,0 +1,217 @@ + + */ +class AgentMapper extends QBMapper { + public function __construct(IDBConnection $db) { + parent::__construct($db, 'assistant_agents', Agent::class); + } + + /** + * @param int $id + * @return Agent + * @throws DoesNotExistException + * @throws MultipleObjectsReturnedException + * @throws Exception + */ + public function getById(int $id): Agent { + $qb = $this->db->getQueryBuilder(); + $qb->select(Agent::$columns) + ->from($this->getTableName()) + ->where($qb->expr()->eq('id', $qb->createPositionalParameter($id, IQueryBuilder::PARAM_INT))); + return $this->findEntity($qb); + } + + /** + * @return list + * @throws Exception + */ + public function getAllAgents(): array { + $qb = $this->db->getQueryBuilder(); + $qb->select(Agent::$columns) + ->from($this->getTableName()) + ->orderBy('id', 'ASC'); + return $this->findEntities($qb); + } + + /** + * @param string $userId + * @return Agent + * @throws DoesNotExistException + * @throws MultipleObjectsReturnedException + * @throws Exception + */ + public function getByUserId(string $userId): Agent { + $qb = $this->db->getQueryBuilder(); + $qb->select(Agent::$columns) + ->from($this->getTableName()) + ->where($qb->expr()->eq('user_id', $qb->createPositionalParameter($userId, IQueryBuilder::PARAM_STR))); + return $this->findEntity($qb); + } + + /** + * Whether a list of user ids contains registered agents, keyed by user id + * + * @param list $userIds + * @return array + * @throws Exception + */ + public function getByUserIds(array $userIds): array { + if ($userIds === []) { + return []; + } + $qb = $this->db->getQueryBuilder(); + $qb->select(Agent::$columns) + ->from($this->getTableName()) + ->where($qb->expr()->in('user_id', $qb->createParameter('userIds'))) + ->setParameter('userIds', $userIds, IQueryBuilder::PARAM_STR_ARRAY); + $result = $qb->executeQuery(); + $agents = []; + while (($row = $result->fetchAssociative()) !== false) { + $agent = $this->mapRowToEntity($row); + $agents[$agent->getUserId()] = $agent; + } + $result->closeCursor(); + return $agents; + } + + /** + * @param Agent $agent + * @return Agent + * @throws Exception + */ + public function deleteAgent(Agent $agent): Agent { + $this->delete($agent); + + $qb = $this->db->getQueryBuilder(); + $qb->delete('assistant_agents_acl') + ->where($qb->expr()->eq('agent_id', $qb->createPositionalParameter($agent->getId(), IQueryBuilder::PARAM_INT))); + $qb->executeStatement(); + + return $agent; + } + + /** + * The access control entries of an agent, as a list of {type, target} + * + * @param int $agentId + * @return list + * @throws Exception + */ + public function getAcl(int $agentId): array { + $qb = $this->db->getQueryBuilder(); + $qb->select('type', 'target') + ->from('assistant_agents_acl') + ->where($qb->expr()->eq('agent_id', $qb->createPositionalParameter($agentId, IQueryBuilder::PARAM_INT))) + ->orderBy('id', 'ASC'); + $result = $qb->executeQuery(); + + $entries = []; + while (($row = $result->fetchAssociative()) !== false) { + $entries[] = [ + 'type' => (string)$row['type'], + 'target' => (string)$row['target'], + ]; + } + $result->closeCursor(); + return $entries; + } + + /** + * Replace the access control entries of an agent + * + * @param int $agentId + * @param list $entries + * @return void + * @throws Exception + */ + public function setAcl(int $agentId, array $entries): void { + $this->db->beginTransaction(); + try { + $qb = $this->db->getQueryBuilder(); + $qb->delete('assistant_agents_acl') + ->where($qb->expr()->eq('agent_id', $qb->createPositionalParameter($agentId, IQueryBuilder::PARAM_INT))); + $qb->executeStatement(); + + foreach ($entries as $entry) { + $qb = $this->db->getQueryBuilder(); + $qb->insert('assistant_agents_acl') + ->values([ + 'agent_id' => $qb->createPositionalParameter($agentId, IQueryBuilder::PARAM_INT), + 'type' => $qb->createPositionalParameter($entry['type'], IQueryBuilder::PARAM_STR), + 'target' => $qb->createPositionalParameter($entry['target'], IQueryBuilder::PARAM_STR), + ]); + $qb->executeStatement(); + } + $this->db->commit(); + } catch (\Throwable $e) { + $this->db->rollBack(); + throw new Exception('Failed to store the agent access list: ' . $e->getMessage(), $e instanceof Exception ? $e->getCode() : 0, $e); + } + } + + /** + * Whether the user (by own id or group membership) matches an entry of the agent ACL. + * An empty ACL means every user has access. + * + * @param int $agentId + * @param string $userId + * @param list $groupIds + * @return bool + * @throws Exception + */ + public function canUserAccess(int $agentId, string $userId, array $groupIds): bool { + $qb = $this->db->getQueryBuilder(); + $accessConditions = $qb->expr()->orX(); + $accessConditions->add($qb->expr()->andX( + $qb->expr()->eq('type', $qb->createNamedParameter('user', IQueryBuilder::PARAM_STR)), + $qb->expr()->eq('target', $qb->createNamedParameter($userId, IQueryBuilder::PARAM_STR)), + )); + if ($groupIds !== []) { + $accessConditions->add($qb->expr()->andX( + $qb->expr()->eq('type', $qb->createNamedParameter('group', IQueryBuilder::PARAM_STR)), + $qb->expr()->in('target', $qb->createNamedParameter($groupIds, IQueryBuilder::PARAM_STR_ARRAY)), + )); + } + + $qb->select('id') + ->from('assistant_agents_acl') + ->where($qb->expr()->eq('agent_id', $qb->createNamedParameter($agentId, IQueryBuilder::PARAM_INT))) + ->andWhere($accessConditions) + ->setMaxResults(1); + $result = $qb->executeQuery(); + $hasAccess = $result->fetchOne() !== false; + $result->closeCursor(); + if ($hasAccess) { + return true; + } + + // an agent with an empty access list is accessible to every user + $qb = $this->db->getQueryBuilder(); + $qb->select('id') + ->from('assistant_agents_acl') + ->where($qb->expr()->eq('agent_id', $qb->createNamedParameter($agentId, IQueryBuilder::PARAM_INT))) + ->setMaxResults(1); + $result = $qb->executeQuery(); + $hasRestrictions = $result->fetchOne() !== false; + $result->closeCursor(); + return !$hasRestrictions; + } +} diff --git a/lib/Db/ChattyLLM/Session.php b/lib/Db/ChattyLLM/Session.php index 1541aaac..1dd45d1b 100644 --- a/lib/Db/ChattyLLM/Session.php +++ b/lib/Db/ChattyLLM/Session.php @@ -27,6 +27,20 @@ * @method \void setAgencyPendingActions(?string $agencyPendingActions) * @method \void setAssignmentId(?int $id) * @method \int|null getAssignmentId() + * @method \string|null getRequesterUserId() + * @method \void setRequesterUserId(?string $requesterUserId) + * @method \string|null getRoomToken() + * @method \void setRoomToken(?string $roomToken) + * @method \int|null getThreadId() + * @method \void setThreadId(?int $threadId) + * @method \string|null getAgentSeenRoomToken() + * @method \void setAgentSeenRoomToken(?string $agentSeenRoomToken) + * @method \int|null getAgentSeenCommentId() + * @method \void setAgentSeenCommentId(?int $agentSeenCommentId) + * @method \string|null getAgencyConfirmationRoomToken() + * @method \void setAgencyConfirmationRoomToken(?string $agencyConfirmationRoomToken) + * @method \int|null getAgencyConfirmationCommentId() + * @method \void setAgencyConfirmationCommentId(?int $agencyConfirmationCommentId) */ class Session extends Entity implements \JsonSerializable { /** @var string */ @@ -63,6 +77,46 @@ class Session extends Entity implements \JsonSerializable { */ protected $assignmentId; + /** + * When the session is owned by an agent (agent-as-a-user feature): the NC user id + * of the human user the agent converses with + * @var ?string + */ + protected $requesterUserId; + + /** + * When the session is used by an agent through Talk: the token of the 1-1 Talk room + * @var ?string + */ + protected $roomToken; + + /** + * When the session is used by an agent through Talk: the id of the Talk thread for this session + * @var ?int + */ + protected $threadId; + + /** + * The Talk room + comment of the last 👀 reaction of the agent on a human message, + * replaced by ✅ or 🔴 once the requested message generation completed/failed + * @var ?string + */ + protected $agentSeenRoomToken; + + /** @var ?int */ + protected $agentSeenCommentId; + + /** + * The Talk room + comment of the pending confirmation message + * (sensitive actions of the agency feature), used to remove the + * 👍/👎 reactions when the confirmation is answered implicitly + * @var ?string + */ + protected $agencyConfirmationRoomToken; + + /** @var ?int */ + protected $agencyConfirmationCommentId; + public static $columns = [ 'id', 'user_id', @@ -73,7 +127,13 @@ class Session extends Entity implements \JsonSerializable { 'summary', 'is_summary_up_to_date', 'is_remembered', - 'assignment_id' + 'assignment_id', + 'requester_user_id', 'room_token', + 'thread_id', + 'agent_seen_room_token', + 'agent_seen_comment_id', + 'agency_confirmation_room_token', + 'agency_confirmation_comment_id', ]; public static $fields = [ 'id', @@ -85,7 +145,14 @@ class Session extends Entity implements \JsonSerializable { 'summary', 'isSummaryUpToDate', 'isRemembered', - 'assignmentId' + 'assignmentId', + 'requesterUserId', + 'roomToken', + 'threadId', + 'agentSeenRoomToken', + 'agentSeenCommentId', + 'agencyConfirmationRoomToken', + 'agencyConfirmationCommentId', ]; public function __construct() { @@ -98,6 +165,13 @@ public function __construct() { $this->addType('isSummaryUpToDate', Types::SMALLINT); $this->addType('isRemembered', Types::SMALLINT); $this->addType('assignmentId', Types::BIGINT); + $this->addType('requesterUserId', Types::STRING); + $this->addType('roomToken', Types::STRING); + $this->addType('threadId', Types::BIGINT); + $this->addType('agentSeenRoomToken', Types::STRING); + $this->addType('agentSeenCommentId', Types::BIGINT); + $this->addType('agencyConfirmationRoomToken', Types::STRING); + $this->addType('agencyConfirmationCommentId', Types::BIGINT); } #[\ReturnTypeWillChange] @@ -113,6 +187,13 @@ public function jsonSerialize() { 'is_summary_up_to_date' => $this->getIsSummaryUpToDate(), 'is_remembered' => $this->getIsRemembered(), 'assignment_id' => $this->getAssignmentId(), + 'requester_user_id' => $this->getRequesterUserId(), + 'room_token' => $this->getRoomToken(), + 'thread_id' => $this->getThreadId(), + 'agent_seen_room_token' => $this->getAgentSeenRoomToken(), + 'agent_seen_comment_id' => $this->getAgentSeenCommentId(), + 'agency_confirmation_room_token' => $this->getAgencyConfirmationRoomToken(), + 'agency_confirmation_comment_id' => $this->getAgencyConfirmationCommentId(), ]; } diff --git a/lib/Db/ChattyLLM/SessionMapper.php b/lib/Db/ChattyLLM/SessionMapper.php index 5c089c9b..39c2f49a 100644 --- a/lib/Db/ChattyLLM/SessionMapper.php +++ b/lib/Db/ChattyLLM/SessionMapper.php @@ -82,6 +82,67 @@ public function getUserSessionForAssignment(string $userId, int $assignmentId): return $this->findEntity($qb); } + /** + * Get the agent session owned by the given agent user for the requester + * in the origin Talk room. + * + * @param string $userId The agent user id (owner of the session) + * @param string $requesterUserId + * @param string $roomToken + * @return Session + * @throws DoesNotExistException + * @throws MultipleObjectsReturnedException + * @throws Exception + */ + public function getAgentSession(string $userId, string $requesterUserId, string $roomToken): Session { + $qb = $this->db->getQueryBuilder(); + $qb->select(Session::$columns) + ->from($this->getTableName()) + ->where($qb->expr()->eq('user_id', $qb->createPositionalParameter($userId, IQueryBuilder::PARAM_STR))) + ->andWhere($qb->expr()->eq('requester_user_id', $qb->createPositionalParameter($requesterUserId, IQueryBuilder::PARAM_STR))) + ->andWhere($qb->expr()->eq('room_token', $qb->createPositionalParameter($roomToken, IQueryBuilder::PARAM_STR))) + ->orderBy('timestamp', 'DESC') + ->setMaxResults(1); + return $this->findEntity($qb); + } + + /** + * Get the agent session owned by the given agent user that posts in the given Talk thread. + * + * @param string $userId The agent user id (owner of the session) + * @param int $threadId The Talk thread id (same as the id of the thread starter Talk comment) + * @return Session + * @throws DoesNotExistException + * @throws MultipleObjectsReturnedException + * @throws Exception + */ + public function getAgentSessionByThread(string $userId, int $threadId): Session { + $qb = $this->db->getQueryBuilder(); + $qb->select(Session::$columns) + ->from($this->getTableName()) + ->where($qb->expr()->eq('user_id', $qb->createPositionalParameter($userId, IQueryBuilder::PARAM_STR))) + ->andWhere($qb->expr()->eq('thread_id', $qb->createPositionalParameter($threadId, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->isNotNull('requester_user_id')) + ->orderBy('timestamp', 'DESC') + ->setMaxResults(1); + return $this->findEntity($qb); + } + + /** + * @param int $sessionId + * @return Session + * @throws DoesNotExistException + * @throws MultipleObjectsReturnedException + * @throws Exception + */ + public function getSessionById(int $sessionId): Session { + $qb = $this->db->getQueryBuilder(); + $qb->select(Session::$columns) + ->from($this->getTableName()) + ->where($qb->expr()->eq('id', $qb->createPositionalParameter($sessionId, IQueryBuilder::PARAM_INT))); + return $this->findEntity($qb); + } + /** * @return \Generator * @throws \OCP\DB\Exception diff --git a/lib/Listener/ChattyLLMTaskListener.php b/lib/Listener/ChattyLLMTaskListener.php index c1dcc058..c18e3ed9 100644 --- a/lib/Listener/ChattyLLMTaskListener.php +++ b/lib/Listener/ChattyLLMTaskListener.php @@ -13,6 +13,7 @@ use OCA\Assistant\Db\ChattyLLM\Message; use OCA\Assistant\Db\ChattyLLM\MessageMapper; use OCA\Assistant\Db\ChattyLLM\SessionMapper; +use OCA\Assistant\Service\AgentTalkService; use OCA\Assistant\Service\NotificationService; use OCA\Assistant\Service\TaskProcessingService; use OCP\AppFramework\Db\DoesNotExistException; @@ -32,6 +33,7 @@ public function __construct( private MessageMapper $messageMapper, private SessionMapper $sessionMapper, private TaskProcessingService $taskProcessingService, + private AgentTalkService $agentTalkService, private LoggerInterface $logger, private NotificationService $notificationService, ) { @@ -43,13 +45,27 @@ public function handle(Event $event): void { $customId = $task->getCustomId(); if (preg_match('/^chatty-llm:(\d+)/', $customId, $matches)) { $sessionId = (int)$matches[1]; - $session = $this->sessionMapper->getUserSession($task->getUserId(), $sessionId); + try { + $session = $this->sessionMapper->getUserSession($task->getUserId(), $sessionId); + } catch (\OCP\AppFramework\Db\DoesNotExistException) { + return; + } catch (\OCP\AppFramework\Db\MultipleObjectsReturnedException|\OCP\DB\Exception $e) { + $this->logger->error('Could not load the session of a failed chat task', ['exception' => $e]); + return; + } $assignmentId = $session->getAssignmentId(); if ($assignmentId !== null) { $this->notificationService->sendAssignmentNotification( $task->getUserId(), $task, $session ); } + // in the agent-as-a-user feature, the 👀 reaction of the last message + // becomes 🔴 when the generation failed + try { + $this->agentTalkService->markGenerationFailed($task->getUserId(), $sessionId); + } catch (\Throwable $e) { + $this->logger->error('Failed to mark the agent message as failed in Talk', ['exception' => $e]); + } } return; } @@ -99,6 +115,31 @@ public function handle(Event $event): void { $taskOutput = $task->getOutput(); + // the transcript of the human's audio input, posted in Talk for agent + // sessions; not on confirmation follow-ups, where the same audio was + // already transcribed when it was first sent. The generated speech + // audio of the answer is posted as a Talk voice message in any case. + $inputTranscript = null; + $audioFileId = null; + if ($isAudioChat || $isAgencyAudioChat) { + $taskInput = $task->getInput(); + $isConfirmation = isset($taskInput['confirmation']) && (int)$taskInput['confirmation'] !== 0; + $inputTranscript = $isConfirmation ? null : trim($taskOutput['input_transcript'] ?? ''); + if (trim($taskOutput['output_transcript'] ?? '') !== '' && is_numeric($taskOutput['output'] ?? null)) { + $audioFileId = (int)$taskOutput['output']; + } + } + // the generated files of a multimodal answer, posted as Talk file shares + $outputFileIds = []; + if ($isMultimodalChat || $isMultimodalAgencyChat) { + $outputAttachments = $taskOutput['output_attachments'] ?? []; + foreach (is_array($outputAttachments) ? $outputAttachments : [] as $outputAttachment) { + if (is_numeric($outputAttachment)) { + $outputFileIds[] = (int)$outputAttachment; + } + } + } + $message = new Message(); $message->setSessionId($sessionId); $message->setOcpTaskId($task->getId()); @@ -180,6 +221,19 @@ public function handle(Event $event): void { } $this->sessionMapper->update($session); + + // in the agent-as-a-user feature, the generated message is posted + // in the 1-1 Talk conversation with the human user + if ($session->getRequesterUserId() !== null) { + $this->agentTalkService->postAgentTalkReply( + $session, + $message->getContent(), + $taskOutput['actions'] ?? null, + $inputTranscript !== '' ? $inputTranscript : null, + $audioFileId, + $outputFileIds, + ); + } } } diff --git a/lib/Listener/Talk/AgentReactionListener.php b/lib/Listener/Talk/AgentReactionListener.php new file mode 100644 index 00000000..2dd38952 --- /dev/null +++ b/lib/Listener/Talk/AgentReactionListener.php @@ -0,0 +1,70 @@ + + * + * Handles the reactions of human users on confirmation-related agent Talk messages: + * they are sent back to the agent context as approve/reject signals. + * + * @psalm-suppress InvalidTemplateParam, ImplementedParamTypeMismatch + */ +class AgentReactionListener implements IEventListener { + + public function __construct( + private AgentTalkService $agentTalkService, + private \OCA\Assistant\Service\AgentsService $agentsService, + private LoggerInterface $logger, + ) { + } + + /** + * @inheritDoc + */ + public function handle(Event $event): void { + if (!class_exists('OCA\Talk\Events\ReactionAddedEvent') + || !$event instanceof \OCA\Talk\Events\ReactionAddedEvent) { + return; + } + try { + $reaction = $event->getReaction(); + if ($event->getActorType() !== 'users') { + return; + } + // ignore reactions added by agent users themselves + if ($this->agentsService->getAgentByUserId($event->getActorId()) !== null) { + return; + } + $comment = $event->getMessage(); + $referenceId = $comment->getReferenceId(); + if (!preg_match('/^' . AgentTalkService::CONFIRMATION_REFERENCE_PREFIX . '(\d+)(:[0-9]+)?$/', $referenceId ?? '', $matches)) { + return; + } + if (!in_array($reaction, [AgentTalkService::REACTION_APPROVE, AgentTalkService::REACTION_REJECT], true)) { + return; + } + $this->agentTalkService->handleConfirmationReaction( + $event->getActorId(), + (int)$matches[1], + (int)$comment->getId(), + $reaction, + $event->getRoom(), + ); + } catch (\Throwable $e) { + $this->logger->error('Failed to react to a Talk reaction for an agent', ['exception' => $e]); + } + } +} diff --git a/lib/Listener/Talk/AgentTalkMessageListener.php b/lib/Listener/Talk/AgentTalkMessageListener.php new file mode 100644 index 00000000..a57cec88 --- /dev/null +++ b/lib/Listener/Talk/AgentTalkMessageListener.php @@ -0,0 +1,62 @@ + + * + * Handles messages sent by users in Talk conversations with agents: + * - chat messages (mentions in group rooms, direct messages in 1-1 + * conversations) come as ChatMessageSentEvent + * - file shares (voice messages, attachments) come as SystemMessageSentEvent: + * they are "file_shared" system messages + * + * The agent answer is generated in a classic Assistant chat session + * and posts back in the 1-1 Talk conversation with the human user. + * + * @psalm-suppress InvalidTemplateParam, ImplementedParamTypeMismatch + */ +class AgentTalkMessageListener implements IEventListener { + + public function __construct( + private AgentTalkService $agentTalkService, + private LoggerInterface $logger, + ) { + } + + /** + * @inheritDoc + */ + public function handle(Event $event): void { + $room = null; + $comment = null; + if (class_exists('OCA\Talk\Events\ChatMessageSentEvent') + && $event instanceof \OCA\Talk\Events\ChatMessageSentEvent) { + $room = $event->getRoom(); + $comment = $event->getComment(); + } elseif (class_exists('OCA\Talk\Events\SystemMessageSentEvent') + && $event instanceof \OCA\Talk\Events\SystemMessageSentEvent) { + $room = $event->getRoom(); + $comment = $event->getComment(); + } else { + return; + } + try { + $this->agentTalkService->handleIncomingTalkMessage($room, $comment); + } catch (\Throwable $e) { + $this->logger->error('Failed to react to a Talk message for an agent', ['exception' => $e]); + } + } +} diff --git a/lib/Migration/Version040000Date20261023083738.php b/lib/Migration/Version040000Date20261023083738.php new file mode 100644 index 00000000..7626e0e5 --- /dev/null +++ b/lib/Migration/Version040000Date20261023083738.php @@ -0,0 +1,88 @@ +hasTable('assistant_agents')) { + $table = $schema->createTable('assistant_agents'); + $table->addColumn('id', Types::BIGINT, [ + 'autoincrement' => true, + 'notnull' => true, + 'length' => 20, + 'unsigned' => true, + ]); + $table->addColumn('user_id', Types::STRING, [ + 'notnull' => true, + 'length' => 64, + ]); + $table->addColumn('personality', Types::TEXT, [ + 'notnull' => false, + ]); + $table->addColumn('scope', Types::TEXT, [ + 'notnull' => false, + ]); + $table->setPrimaryKey(['id']); + $table->addUniqueIndex(['user_id'], 'assistant_agents_user_id'); + $schemaChanged = true; + } + + if ($schema->hasTable('assistant_chat_sns')) { + $table = $schema->getTable('assistant_chat_sns'); + // agent sessions are owned by the agent user account, + // requester_user_id is the id of the human user the agent converses with + if (!$table->hasColumn('requester_user_id')) { + $table->addColumn('requester_user_id', Types::STRING, [ + 'notnull' => false, + 'length' => 64, + ]); + $schemaChanged = true; + } + if (!$table->hasColumn('room_token')) { + $table->addColumn('room_token', Types::STRING, [ + 'notnull' => false, + 'length' => 64, + ]); + $schemaChanged = true; + } + if (!$table->hasColumn('thread_id')) { + $table->addColumn('thread_id', Types::BIGINT, [ + 'notnull' => false, + 'length' => 20, + 'unsigned' => true, + ]); + $schemaChanged = true; + } + if (!$table->hasIndex('assistant_chat_requester_idx')) { + $table->addIndex(['user_id', 'requester_user_id', 'room_token'], 'assistant_chat_requester_idx'); + $schemaChanged = true; + } + } + + return $schemaChanged ? $schema : null; + } +} diff --git a/lib/Migration/Version040000Date20261023120000.php b/lib/Migration/Version040000Date20261023120000.php new file mode 100644 index 00000000..a9c8ea6a --- /dev/null +++ b/lib/Migration/Version040000Date20261023120000.php @@ -0,0 +1,51 @@ +hasTable('assistant_chat_sns')) { + $table = $schema->getTable('assistant_chat_sns'); + // the Talk room + message of the last 👀 reaction of the agent, + // so it can be replaced by the generated answer state (✅/🔴) + if (!$table->hasColumn('agent_seen_room_token')) { + $table->addColumn('agent_seen_room_token', Types::STRING, [ + 'notnull' => false, + 'length' => 64, + ]); + $table->addColumn('agent_seen_comment_id', Types::BIGINT, [ + 'notnull' => false, + 'length' => 20, + 'unsigned' => true, + ]); + $schemaChanged = true; + } + } + + return $schemaChanged ? $schema : null; + } +} diff --git a/lib/Migration/Version040100Date20261113120000.php b/lib/Migration/Version040100Date20261113120000.php new file mode 100644 index 00000000..32d6ba22 --- /dev/null +++ b/lib/Migration/Version040100Date20261113120000.php @@ -0,0 +1,52 @@ +hasTable('assistant_chat_sns')) { + $table = $schema->getTable('assistant_chat_sns'); + // the Talk room + message of the pending confirmation + // so the 👍/👎 reactions can be removed when the confirmation + // is answered implicitly by sending a new message + if (!$table->hasColumn('agency_confirmation_room_token')) { + $table->addColumn('agency_confirmation_room_token', Types::STRING, [ + 'notnull' => false, + 'length' => 64, + ]); + $table->addColumn('agency_confirmation_comment_id', Types::BIGINT, [ + 'notnull' => false, + 'length' => 20, + 'unsigned' => true, + ]); + $schemaChanged = true; + } + } + + return $schemaChanged ? $schema : null; + } +} diff --git a/lib/Migration/Version040200Date20261113130000.php b/lib/Migration/Version040200Date20261113130000.php new file mode 100644 index 00000000..99bf7ead --- /dev/null +++ b/lib/Migration/Version040200Date20261113130000.php @@ -0,0 +1,59 @@ +hasTable('assistant_agents_acl')) { + return null; + } + + $table = $schema->createTable('assistant_agents_acl'); + $table->addColumn('id', Types::BIGINT, [ + 'autoincrement' => true, + 'notnull' => true, + 'length' => 20, + 'unsigned' => true, + ]); + $table->addColumn('agent_id', Types::BIGINT, [ + 'notnull' => true, + 'length' => 20, + 'unsigned' => true, + ]); + $table->addColumn('type', Types::STRING, [ + 'notnull' => true, + 'length' => 10, + ]); + // the uid or the gid depending on the type + $table->addColumn('target', Types::STRING, [ + 'notnull' => true, + 'length' => 64, + ]); + $table->setPrimaryKey(['id']); + $table->addIndex(['agent_id'], 'assistant_agents_acl_agent'); + return $schema; + } +} diff --git a/lib/Service/AgencyActionService.php b/lib/Service/AgencyActionService.php new file mode 100644 index 00000000..4c2eb293 --- /dev/null +++ b/lib/Service/AgencyActionService.php @@ -0,0 +1,247 @@ +actionData = [ + // talk + 'send_message_to_conversation' => [ + 'title' => $this->l10n->t('Send a message to a Talk conversation'), + 'icon' => 'Send', + ], + 'create_public_conversation' => [ + 'title' => $this->l10n->t('Create a conversation'), + 'icon' => 'ChatPlus', + ], + // mail + 'send_email' => [ + 'title' => $this->l10n->t('Send an email'), + 'icon' => 'EmailPlus', + ], + // calendar + 'schedule_event' => [ + 'title' => $this->l10n->t('Schedule a calendar event'), + 'icon' => 'CalendarPlus', + ], + 'add_task' => [ + 'title' => $this->l10n->t('Add a calendar task'), + 'icon' => 'CalendarCheck', + ], + // deck + 'add_card' => [ + 'title' => $this->l10n->t('Create a Deck card'), + 'icon' => 'CardPlus', + ], + 'upload_file' => [ + 'title' => $this->l10n->t('Upload file'), + 'icon' => 'Upload', + ], + 'create_folder' => [ + 'title' => $this->l10n->t('Create folder'), + 'icon' => 'FolderPlus', + ], + 'move_file' => [ + 'title' => $this->l10n->t('Move file'), + 'icon' => 'FileMove', + ], + 'copy_file' => [ + 'title' => $this->l10n->t('Copy file'), + 'icon' => 'FileCopy', + ], + 'delete_file' => [ + 'title' => $this->l10n->t('Delete file'), + 'icon' => 'Delete', + ], + 'create_bookmark' => [ + 'title' => $this->l10n->t('Create bookmark'), + 'icon' => 'BookmarkPlus', + ], + 'update_bookmark' => [ + 'title' => $this->l10n->t('Update bookmark'), + 'icon' => 'BookmarkEdit', + ], + 'delete_bookmark' => [ + 'title' => $this->l10n->t('Delete bookmark'), + 'icon' => 'BookmarkRemove', + ], + 'create_bookmark_folder' => [ + 'title' => $this->l10n->t('Create bookmark folder'), + 'icon' => 'FolderPlus', + ], + 'create_circle' => [ + 'title' => $this->l10n->t('Create team'), + 'icon' => 'AccountGroupPlus', + ], + 'add_member_to_circle' => [ + 'title' => $this->l10n->t('Add member to team'), + 'icon' => 'AccountPlus', + ], + 'remove_member_from_circle' => [ + 'title' => $this->l10n->t('Remove member from team'), + 'icon' => 'AccountRemove', + ], + 'update_circle' => [ + 'title' => $this->l10n->t('Update team'), + 'icon' => 'AccountGroupEdit', + ], + 'delete_circle' => [ + 'title' => $this->l10n->t('Delete team'), + 'icon' => 'AccountGroupRemove', + ], + 'share_with_circle' => [ + 'title' => $this->l10n->t('Share with team'), + 'icon' => 'ShareVariant', + ], + 'create_form' => [ + 'title' => $this->l10n->t('Create form'), + 'icon' => 'FormTextbox', + ], + 'add_question_to_form' => [ + 'title' => $this->l10n->t('Add question to form'), + 'icon' => 'CommentPlus', + ], + 'delete_form' => [ + 'title' => $this->l10n->t('Delete form'), + 'icon' => 'Delete', + ], + 'update_form_settings' => [ + 'title' => $this->l10n->t('Update form settings'), + 'icon' => 'Cog', + ], + 'share_with_user' => [ + 'title' => $this->l10n->t('Share with user'), + 'icon' => 'AccountShare', + ], + 'share_with_group' => [ + 'title' => $this->l10n->t('Share with group'), + 'icon' => 'AccountGroup', + ], + 'update_share_permissions' => [ + 'title' => $this->l10n->t('Update share permissions'), + 'icon' => 'LockOpen', + ], + 'delete_share' => [ + 'title' => $this->l10n->t('Delete share'), + 'icon' => 'Delete', + ], + ]; + } + + public function getActionTitle(string $toolName): ?string { + return $this->actionData[$toolName]['title'] ?? null; + } + + public function getActionIcon(string $toolName): ?string { + return $this->actionData[$toolName]['icon'] ?? null; + } + + /** + * Convert the pending actions JSON of a generation task output into a + * natural language confirmation message for Talk. + * + * Returns null when the actions JSON is empty or cannot be decoded; + * callers fall back to posting the raw string in that case. + */ + public function formatConfirmationMessage(string $pendingActions): ?string { + $actions = json_decode($pendingActions, true); + if (!is_array($actions) || $actions === []) { + return null; + } + + $formattedActions = []; + foreach ($actions as $action) { + $formatted = $this->formatAction($action); + if ($formatted !== null) { + $formattedActions[] = $formatted; + } + } + if ($formattedActions === []) { + return null; + } + + $blocks = []; + foreach ($formattedActions as $index => $action) { + $lines = []; + if ($action['title'] !== null) { + $prefix = count($formattedActions) > 1 ? ($index + 1) . '. ' : ''; + $lines[] = $prefix . '**' . $action['title'] . '**'; + } + $blocks[] = implode("\n", array_merge($lines, $action['paramLines'])); + } + + if (count($formattedActions) === 1) { + return $this->l10n->t('I want to perform a sensitive action:') . "\n\n" . $blocks[0]; + } + + return $this->l10n->t('I want to perform the following sensitive actions:') + . "\n\n" . implode("\n\n", $blocks); + } + + /** + * @return array{title: ?string, paramLines: string[]}|null null when the action has neither a name nor arguments + */ + private function formatAction(mixed $action): ?array { + if (!is_array($action)) { + return null; + } + $title = $this->getActionDisplayTitle($action); + $paramLines = []; + foreach ($action['args'] ?? [] as $argName => $argValue) { + $paramLines[] = '* ' . str_replace('_', ' ', (string)$argName) . ': ' . $this->formatParamValue($argValue); + } + if ($title === null && $paramLines === []) { + return null; + } + return ['title' => $title, 'paramLines' => $paramLines]; + } + + private function getActionDisplayTitle(array $action): ?string { + $toolName = $action['name'] ?? ''; + if (!is_string($toolName) || $toolName === '') { + return null; + } + return $this->getActionTitle($toolName) ?? $this->prettifyToolName($toolName); + } + + private function prettifyToolName(string $toolName): string { + return ucfirst(str_replace('_', ' ', $toolName)); + } + + private function formatParamValue(mixed $value): string { + if (is_array($value)) { + if (array_is_list($value) && array_reduce($value, fn ($ok, $v) => $ok && (is_string($v) || is_numeric($v)), true)) { + // a plain list of values is rendered as a comma-separated list + return implode(', ', array_map(fn ($v) => $this->formatParamValue($v), $value)); + } + return json_encode($value) ?: ''; + } + if (is_bool($value)) { + return $value ? 'true' : 'false'; + } + if ($value === null) { + return ''; + } + // newlines would break the single markdown list item structure + return trim(str_replace(["\r", "\n"], ' ', (string)$value)); + } +} diff --git a/lib/Service/AgentTalkFileService.php b/lib/Service/AgentTalkFileService.php new file mode 100644 index 00000000..a464bf21 --- /dev/null +++ b/lib/Service/AgentTalkFileService.php @@ -0,0 +1,232 @@ +getMessage(), true); + if (!is_array($decoded) + || ($decoded['message'] ?? '') !== self::VERB_MESSAGE_FILE_SHARED + || !isset($decoded['parameters']) + || !is_array($decoded['parameters'])) { + return null; + } + $parameters = $decoded['parameters']; + + $fileId = null; + if (isset($parameters['fileId']) && is_numeric($parameters['fileId'])) { + $fileId = (int)$parameters['fileId']; + } + $shareId = null; + if (isset($parameters['share']) && is_numeric($parameters['share'])) { + $shareId = (string)$parameters['share']; + } + if ($fileId === null && $shareId === null) { + return null; + } + + $metaData = $parameters['metaData'] ?? []; + if (!is_array($metaData)) { + $metaData = []; + } + + return [ + 'fileId' => $fileId, + 'shareId' => $shareId, + 'messageType' => isset($metaData['messageType']) ? (string)$metaData['messageType'] : null, + 'mimeType' => isset($metaData['mimeType']) ? (string)$metaData['mimeType'] : null, + 'caption' => isset($metaData['caption']) && is_string($metaData['caption']) ? trim($metaData['caption']) : '', + ]; + } + + /** + * The chat message attachment type for a Talk file share: audio files + * (voice messages, call recordings) are routed to the audio chat tasks, + * all other files to the multimodal chat tasks. + * + * @param array{messageType: ?string, mimeType: ?string} $fileShare + */ + public function getAttachmentType(array $fileShare, ?Node $node): string { + $mimeType = $fileShare['mimeType']; + if ($mimeType === null && $node instanceof File) { + $mimeType = $node->getMimeType(); + } + if ($fileShare['messageType'] === self::MESSAGE_TYPE_VOICE_MESSAGE + || ($mimeType !== null && str_starts_with($mimeType, 'audio/'))) { + return self::ATTACHMENT_TYPE_AUDIO; + } + return self::ATTACHMENT_TYPE_FILE; + } + + /** + * The mentions written in the caption of a Talk file share. + * + * Mentions are stored in the caption text, not in the comment message + * (which holds the file share JSON). Like Talk's own notifier, mentions + * are parsed by setting the caption as the comment message. + * + * @return list + */ + public function getCaptionMentions(IComment $comment, array $fileShare): array { + $caption = $fileShare['caption'] ?? ''; + if ($caption === '') { + return []; + } + $captionComment = clone $comment; + $captionComment->setMessage($caption, self::MENTION_PARSE_MAX_LENGTH); + return $captionComment->getMentions(); + } + + /** + * Resolve the node of a Talk file share, or null when the file cannot be + * accessed as an attachment of the room. + * + * The node is looked up in the storage of the sender (a room attachment + * lives in the storage of the user who shared it), and the room must have + * a room share on the node itself or on one of its ancestors: only then + * does the file belong to that Talk room. + * + * @param string $roomToken + * @param string $senderUserId The user id of the file share sender + * @param array{fileId: ?int, shareId: ?string} $fileShare Result of parseFileShareMessage() + */ + public function getRoomAttachmentNode(string $roomToken, string $senderUserId, array $fileShare): ?Node { + $fileId = $fileShare['fileId']; + if ($fileId === null && $fileShare['shareId'] !== null) { + $fileId = $this->getNodeIdFromShare((int)$fileShare['shareId']); + if ($fileId === null) { + return null; + } + } + + try { + $userFolder = $this->rootFolder->getUserFolder($senderUserId); + $candidates = $userFolder->getById($fileId); + } catch (\Throwable $e) { + $this->logger->warning('Could not look up the Talk room attachment in the sender storage', [ + 'fileId' => $fileId, + 'senderUserId' => $senderUserId, + 'exception' => $e, + ]); + return null; + } + + foreach ($candidates as $candidate) { + if (!$candidate instanceof File || !$candidate->isReadable()) { + continue; + } + if ($this->isCoveredByRoomShare($roomToken, $candidate)) { + return $candidate; + } + } + return null; + } + + /** + * @return bool True when a room share for the given token covers the node + * itself or one of its ancestors + */ + private function isCoveredByRoomShare(string $roomToken, Node $node): bool { + /** @var \OCA\Talk\Share\RoomShareProvider $roomShareProvider */ + $roomShareProvider = Server::get(\OCA\Talk\Share\RoomShareProvider::class); + + $current = $node; + for ($depth = 0; $depth < self::MAX_SHARE_ANCESTORS; $depth++) { + try { + $shares = $roomShareProvider->getSharesByPath($current); + } catch (\Throwable $e) { + $this->logger->warning('Could not check the room shares of a Talk attachment', [ + 'nodeId' => $current->getId(), + 'exception' => $e, + ]); + return false; + } + foreach ($shares as $share) { + if ($share->getSharedWith() === $roomToken) { + return true; + } + } + try { + $parent = $current->getParent(); + } catch (NotFoundException $e) { + // the storage root was reached + break; + } + if ($parent->getPath() === $current->getPath()) { + break; + } + $current = $parent; + } + return false; + } + + /** + * @return int|null The node id of the shared file + */ + private function getNodeIdFromShare(int $shareId): ?int { + try { + /** @var \OCA\Talk\Share\RoomShareProvider $roomShareProvider */ + $roomShareProvider = Server::get(\OCA\Talk\Share\RoomShareProvider::class); + return $roomShareProvider->getShareById($shareId)->getNodeId(); + } catch (\Throwable $e) { + $this->logger->warning('Could not resolve the Talk room share of an attachment', [ + 'shareId' => $shareId, + 'exception' => $e, + ]); + return null; + } + } +} diff --git a/lib/Service/AgentTalkService.php b/lib/Service/AgentTalkService.php new file mode 100644 index 00000000..7e76c610 --- /dev/null +++ b/lib/Service/AgentTalkService.php @@ -0,0 +1,1132 @@ +getActorType(); + $authorActorId = $comment->getActorId(); + if ($authorActorType !== 'users') { + return; + } + // Ignore messages sent by agent users to avoid loops + if ($this->agentsService->getAgentByUserId($authorActorId) !== null) { + return; + } + + // only chat messages and file shares are handled; + // other system messages (calls, polls, ...) never trigger an agent + $verb = $comment->getVerb(); + $fileShare = null; + if ($verb === \OCA\Talk\Chat\ChatManager::VERB_OBJECT_SHARED) { + // file shares (voice messages, attachments) are "file_shared" + // system messages; other object shares (polls, ...) are ignored + $fileShare = $this->agentTalkFileService->parseFileShareMessage($comment); + if ($fileShare === null) { + return; + } + } elseif ($verb !== \OCA\Talk\Chat\ChatManager::VERB_MESSAGE) { + return; + } + + $agent = $this->getAgentTarget($room, $comment, $authorActorId, $fileShare); + if ($agent === null) { + return; + } + + $humanUser = $this->userManager->get($authorActorId); + if ($humanUser === null) { + return; + } + + if (!$this->agentsService->canUserAccessAgent($agent, $humanUser)) { + $this->handleAgentAccessDenied($agent, $humanUser, $comment, $room); + return; + } + + $this->processAgentInteraction( + $room, + $agent, + $humanUser, + $comment, + $fileShare, + ); + } catch (\Throwable $e) { + $this->logger->error('Failed to handle a Talk message for an agent', ['exception' => $e]); + } + } + + /** + * Find the agent user targeted by a Talk message: + * - in a 1-1 conversation: the other attendee, if it is a registered agent + * - in a group room: a mentioned registered agent that is a participant of the room. + * Mentions of a file share are stored in its caption, not in the message. + * The agent must always answer in the private 1-1 conversation + * with the user who sent the message. + */ + private function getAgentTarget( + \OCA\Talk\Room $room, + IComment $comment, + string $authorActorId, + ?array $fileShare, + ): ?Agent { + if ($room->getType() === \OCA\Talk\Room::TYPE_ONE_TO_ONE) { + $partnerUserId = $this->getOneToOnePartnerUserId($room, $authorActorId); + if ($partnerUserId !== null) { + return $this->agentsService->getAgentByUserId($partnerUserId); + } + return null; + } + + // In a group room the mentioned agent must already be a participant of the room + /** @var \OCA\Talk\Manager $talkManager */ + $talkManager = Server::get(\OCA\Talk\Manager::class); + $mentions = $comment->getMentions(); + if ($fileShare !== null) { + $mentions = $this->agentTalkFileService->getCaptionMentions($comment, $fileShare); + } + foreach ($mentions as $mention) { + if ($mention['type'] !== 'user') { + continue; + } + $mentionId = $mention['id']; + if ($this->agentsService->getAgentByUserId($mentionId) !== null + && $talkManager->isUserAttendeeInRoom($mentionId, $room->getToken())) { + return $this->agentsService->getAgentByUserId($mentionId); + } + } + return null; + } + + /** + * @return string|null The user id of the other actor in the room + */ + private function getOneToOnePartnerUserId(\OCA\Talk\Room $room, string $authorUserId): ?string { + try { + return $room->getSecondParticipant($authorUserId); + } catch (\InvalidArgumentException|\JsonException|\OCA\Talk\Exceptions\RoomNotFoundException $e) { + return null; + } + } + + /** + * The user does not have access to the agent: no session is created and no + * answer is generated. A short notice is sent as the agent in the private + * 1-1 conversation, as a reply to the triggering message when it was + * written in that 1-1 conversation (or one of its threads). + * + * The reaction lifecycle of the handled message and the pointer message + * posted in a group room are identical to the answer flow, so that room + * members cannot infer who has access to the agent. + */ + private function handleAgentAccessDenied(Agent $agent, IUser $humanUser, IComment $comment, \OCA\Talk\Room $originRoom): void { + if (!class_exists('OCA\Talk\Chat\ChatManager')) { + return; + } + $agentUserId = $agent->getUserId(); + $messageId = (int)$comment->getId(); + + // like the answer flow: 👀 while being handled, ✅ once the notice is posted + $this->addAgentReaction($originRoom, $agentUserId, $messageId, self::REACTION_READ); + try { + $oneToOneRoom = $this->getOrCreateOneToOneRoom($humanUser, $agentUserId); + } catch (AgentUserException|\OCA\Talk\Exceptions\RoomNotFoundException $e) { + $this->logger->warning('Could not get the 1-1 Talk room to post the access notice', [ + 'agentUserId' => $agentUserId, + 'humanUserId' => $humanUser->getUID(), + 'exception' => $e, + ]); + $this->deleteAgentReaction($originRoom, $agentUserId, $messageId, self::REACTION_READ); + $this->addAgentReaction($originRoom, $agentUserId, $messageId, self::REACTION_ERROR); + return; + } + + $content = $this->l10n->t( + 'You do not have access to the agent "%s". Please contact your administrator if you need access.', + [$this->getAgentDisplayName($agent)] + ); + try { + /** @var \OCA\Talk\Chat\ChatManager $chatManager */ + $chatManager = Server::get(\OCA\Talk\Chat\ChatManager::class); + $handledInOneToOne = ($comment->getObjectType() === 'chat' + && $comment->getObjectId() === (string)$oneToOneRoom->getId()); + $noticeComment = $chatManager->sendMessage( + $oneToOneRoom, + null, + 'users', + $agentUserId, + $content, + new DateTime(), + replyTo: $handledInOneToOne ? $comment : null, + ); + } catch (\Throwable $e) { + $this->logger->error('Failed to post the agent access notice in Talk', ['exception' => $e]); + $this->deleteAgentReaction($originRoom, $agentUserId, $messageId, self::REACTION_READ); + $this->addAgentReaction($originRoom, $agentUserId, $messageId, self::REACTION_ERROR); + return; + } + + $this->deleteAgentReaction($originRoom, $agentUserId, $messageId, self::REACTION_READ); + $this->addAgentReaction($originRoom, $agentUserId, $messageId, self::REACTION_DONE); + + if ($originRoom->getType() !== \OCA\Talk\Room::TYPE_ONE_TO_ONE) { + // the mention happened in a group room: post the same pointer + // message as the answer flow, so that room members cannot infer + // who has access to the agent + try { + $this->postRoomPointer($originRoom, $comment, $oneToOneRoom, $noticeComment, $agentUserId); + } catch (\Throwable $e) { + $this->logger->error('Failed to post the access notice pointer message in the origin Talk room', ['exception' => $e]); + } + } + } + + /** + * @throws Exception + * @throws InternalException + * @throws UnauthorizedException + * @throws NotFoundException + * @throws BadRequestException + */ + private function processAgentInteraction( + \OCA\Talk\Room $room, + Agent $agent, + IUser $humanUser, + IComment $comment, + ?array $fileShare, + ): void { + $humanUserId = $humanUser->getUID(); + + // when the message is a reply inside the Talk thread of an agent session, + // the generation happens in that session and the answer goes back to the same thread + if ($room->getType() === \OCA\Talk\Room::TYPE_ONE_TO_ONE) { + $triggerThreadId = (int)($comment->getTopmostParentId() ?? 0); + if ($triggerThreadId !== 0) { + try { + $threadSession = $this->sessionMapper->getAgentSessionByThread($agent->getUserId(), $triggerThreadId); + $this->storeAndSchedule($threadSession, $comment, $room, $agent, $fileShare); + return; + } catch (DoesNotExistException) { + // the reply is not in the thread of an agent session: handle it below + } + } + } + + // the 1-1 room the agent answers in + $this->getOrCreateOneToOneRoom($humanUser, $agent->getUserId()); + + // the name of the context is the origin room of the mention/message, + // or the human display name when it was a direct 1-1 conversation + $contextRoomName = ($room->getType() === \OCA\Talk\Room::TYPE_ONE_TO_ONE) + ? $humanUser->getDisplayName() + : $room->getName(); + + // find or create the assistant chat session dedicated to this agent/origin-room context + $session = $this->getOrCreateAgentSession($humanUserId, $agent, $contextRoomName, $room->getToken()); + + $this->storeAndSchedule($session, $comment, $room, $agent, $fileShare); + } + + /** + * @throws Exception + * @throws InternalException + * @throws UnauthorizedException + * @throws NotFoundException + * @throws BadRequestException + */ + private function storeAndSchedule(Session $session, IComment $comment, \OCA\Talk\Room $room, Agent $agent, ?array $fileShare): void { + // a new human message implicitly answers a pending confirmation + // like a rejection would: remove the now meaningless 👍/👎 reactions + // and the pending actions, then generate a new answer + $this->removePendingConfirmationReactions($session); + $session->setAgencyPendingActions(null); + + // store the human Talk message in the chat session + $creationTimestamp = $comment->getCreationDateTime()->getTimestamp(); + [$talkContent, $attachments] = $this->getTalkMessageContentAndAttachments($comment, $room, $fileShare); + $this->chatService->createMessage( + $session->getUserId(), + (int)$session->getId(), + Message::ROLE_HUMAN, + $talkContent, + $creationTimestamp, + $attachments, + ); + + // mark the message as seen before scheduling, so that the reaction + // also happens when the generation cannot be scheduled right away + // (the session answers one message at a time) + $this->addAgentReaction($room, $agent->getUserId(), (int)$comment->getId(), self::REACTION_READ); + // remember the seen message to later replace 👀 by ✅/🔴 after the generation + $session->setAgentSeenRoomToken($room->getToken()); + $session->setAgentSeenCommentId((int)$comment->getId()); + try { + $this->sessionMapper->update($session); + } catch (Exception $e) { + $this->logger->error('Failed to store the seen message of the agent session', ['exception' => $e]); + } + + // schedule generation of the agent answer + $this->chatService->scheduleMessageGeneration($session->getUserId(), (int)$session->getId(), 0, $this->getOriginRoomInfo($session)); + } + + /** + * Build the chat session content and attachments to store for a Talk message. + * + * A file share (voice message, attachment) is stored as a chat message with + * an attachment referencing the shared file, and its caption (if any) as + * the content; plain messages are stored as they are. + * + * @return array{string, list|null} [content, attachments] + */ + private function getTalkMessageContentAndAttachments(IComment $comment, \OCA\Talk\Room $room, ?array $fileShare): array { + if ($fileShare === null) { + return [(string)$comment->getMessage(), null]; + } + + // the file is read through the storage of the sender, it is an + // attachment of the room only when the room has a share covering it + $node = $this->agentTalkFileService->getRoomAttachmentNode($room->getToken(), $comment->getActorId(), $fileShare); + $caption = $fileShare['caption']; + if ($node === null) { + // the file is not accessible as a room attachment (deleted, revoked): + // fall back to a plain text message + if ($caption !== '') { + return [$caption, null]; + } + return [$this->l10n->t('[A file was shared, but it is no longer available]'), null]; + } + + $attachment = [ + 'type' => $this->agentTalkFileService->getAttachmentType($fileShare, $node), + 'file_id' => $node->getId(), + ]; + return [$caption, [$attachment]]; + } + + /** + * The session is dedicated to one agent + human user + origin Talk room context: + * the Talk room in which the human mentioned the agent or wrote to him directly. + * All the agent answers are posted as one Talk thread of the 1-1 room + * between the human and the agent. + * + * Agent sessions are owned by the agent user account, + * the human user is recorded as the requester. + * + * @throws Exception + */ + private function getOrCreateAgentSession(string $humanUserId, Agent $agent, string $contextRoomName, string $originRoomToken): Session { + try { + return $this->sessionMapper->getAgentSession( + $agent->getUserId(), $humanUserId, $originRoomToken + ); + } catch (DoesNotExistException) { + // continue with session creation + } catch (Exception $e) { + throw new InternalException($e->getMessage(), previous: $e); + } + + $agentName = $this->getAgentDisplayName($agent); + $sessionTitle = $agentName . self::THREAD_TITLE_SEPARATOR . $contextRoomName; + + $session = $this->chatService->createChatSession( + $agent->getUserId(), + timestamp: null, + title: $sessionTitle, + systemPrompt: $agent->getPersonality(), + requesterUserId: $humanUserId, + ); + $session->setRoomToken($originRoomToken); + try { + $this->sessionMapper->update($session); + } catch (Exception $e) { + throw new InternalException('Failed to store the agent context on the chat session', previous: $e); + } + return $session; + } + + public function getAgentDisplayName(Agent $agent): string { + $userId = $agent->getUserId(); + $user = $this->userManager->get($userId); + $displayName = null; + if ($user !== null) { + $displayName = $user->getDisplayName(); + } + return $displayName ?: $userId; + } + + /** + * The 1-1 Talk room between a human user and an agent user + * + * @throws AgentUserException + */ + private function getOrCreateOneToOneRoom(IUser $humanUser, string $agentUserId): \OCA\Talk\Room { + /** @var \OCA\Talk\Manager $talkManager */ + $talkManager = Server::get(\OCA\Talk\Manager::class); + try { + return $talkManager->getOne2OneRoom($humanUser->getUID(), $agentUserId); + } catch (\OCA\Talk\Exceptions\RoomNotFoundException) { + // fall through to the creation below + } + + $agentUser = $this->userManager->get($agentUserId); + if ($agentUser === null) { + throw new AgentUserException('The agent user ' . $agentUserId . ' does not exist'); + } + + /** @var \OCA\Talk\Service\RoomService $talkRoomService */ + $talkRoomService = Server::get(\OCA\Talk\Service\RoomService::class); + try { + return $talkRoomService->createOneToOneConversation($humanUser, $agentUser); + } catch (\OCA\Talk\Exceptions\RoomNotFoundException|\InvalidArgumentException $e) { + throw new AgentUserException('Failed to create the 1-1 conversation with the agent user ' . $agentUserId, previous: $e); + } + } + + /** + * Called by ChattyLLMTaskListener instead of posting the agent answer + * in Talk: it will post the answer in the 1-1 Talk conversation thread. + * + * @param Session $session + * @param string $content The assistant message content + * @param string|null $pendingActions The pending sensitive actions (confirmation request) if any + * @param string|null $inputTranscript The transcript of the human's audio input, if any: + * it is posted before the answer, as the agent + * @param int|null $audioFileId The id of the generated speech audio task output, if any: + * it is posted after the answer as a Talk voice message + * @param list $outputFileIds The ids of the generated files of a multimodal answer: + * they are posted after the answer as Talk file shares + * @return void + */ + public function postAgentTalkReply( + Session $session, + string $content, + ?string $pendingActions, + ?string $inputTranscript = null, + ?int $audioFileId = null, + array $outputFileIds = [], + ): void { + if ($session->getRequesterUserId() === null) { + return; + } + // only for agent sessions + if (!class_exists('OCA\Talk\Chat\ChatManager')) { + return; + } + $agentUser = $this->userManager->get($session->getUserId()); + if ($agentUser === null) { + $this->logger->warning('The agent user was deleted while an agent answer was being handled'); + return; + } + $humanUser = $this->userManager->get($session->getRequesterUserId()); + if ($humanUser === null) { + $this->logger->warning('The requester user was deleted while an agent answer was being handled'); + return; + } + try { + $room = $this->getOrCreateOneToOneRoom($humanUser, $agentUser->getUID()); + } catch (AgentUserException|\OCA\Talk\Exceptions\RoomNotFoundException $e) { + $this->logger->warning('Could not get the 1-1 Talk room to post an agent reply', [ + 'agentuserId' => $agentUser->getUID(), + 'humanUserId' => $humanUser->getUID(), + 'exception' => $e, + ]); + return; + } + + // messages sent outside a thread of a 1-1 conversation are answered + // with plain 1-1 messages; answers to a thread conversation go back in the thread + $threaded = true; + if ($session->getRoomToken() !== null) { + try { + $originRoom = $this->getRoomByToken($session->getRoomToken()); + $threaded = $originRoom->getType() !== \OCA\Talk\Room::TYPE_ONE_TO_ONE; + } catch (\OCA\Talk\Exceptions\RoomNotFoundException) { + // the origin room may have been deleted, still answer threaded + } + } + + // when the human sent a voice message, the agent posts its transcript + // first, so the human sees what was heard, then the answer; + // the transcript is the raw speech-to-text output, not translated + if ($inputTranscript !== null && trim($inputTranscript) !== '') { + $this->postAgentMessage( + $room, + $session, + '🎙️: _"' . trim($inputTranscript) . '"_', + referenceId: null, + threaded: $threaded, + ); + } + + $postedComment = $this->postAgentMessage( + $room, + $session, + $content, + referenceId: null, + threaded: $threaded, + ); + + // generated files of a multimodal answer are posted as Talk file + // shares, then the spoken answer as a voice message + foreach ($outputFileIds as $outputFileId) { + $this->postAgentTalkFileMessage($room, $session, $threaded, (int)$outputFileId, false); + } + if ($audioFileId !== null) { + $this->postAgentTalkFileMessage($room, $session, $threaded, $audioFileId, true); + } + + $confirmPosted = false; + if ($pendingActions !== null && trim($pendingActions) !== '') { + // a new confirmation supersedes a previous unanswered one: + // remove the 👍/👎 reactions of the superseded confirmation message + $this->removePendingConfirmationReactions($session); + // the confirmation request is posted as natural language; + // fall back to the raw actions JSON if it cannot be formatted + $confirmationContent = $this->agencyActionService->formatConfirmationMessage($pendingActions) ?? $pendingActions; + $confirmationComment = $this->postAgentMessage( + $room, + $session, + $confirmationContent, + referenceId: self::CONFIRMATION_REFERENCE_PREFIX . $session->getId() . ':' . time(), + addApproveRejectReactions: true, + threaded: $threaded, + ); + if ($confirmationComment !== null) { + $confirmPosted = true; + // remember the confirmation message to remove the 👍/👎 reactions + // if the user sends a new message instead of reacting + $session->setAgencyConfirmationRoomToken($room->getToken()); + $session->setAgencyConfirmationCommentId((int)$confirmationComment->getId()); + try { + $this->sessionMapper->update($session); + } catch (Exception $e) { + $this->logger->error('Failed to store the pending confirmation of the agent session', ['exception' => $e]); + } + } + } + + if ($postedComment !== null) { + try { + $this->postGroupRoomPointer($session, $room, $postedComment); + } catch (\Throwable $e) { + $this->logger->error('Failed to post the answer pointer message in the origin Talk room', ['exception' => $e]); + } + } + + // an agency confirmation turn has no reply content, only the confirmation message + $this->markGenerationCompleted($session, $postedComment !== null || $confirmPosted); + } + + /** + * Post a generated file of an agent answer (task output) in Talk: + * the file is copied from the task output storage into the agent's + * conversation folder for the room, like Talk's own attachment flow, + * and posted as a "file_shared" message as the agent, in the thread of + * the answer. + * + * Best effort: failures are logged and skipped, the text answer is already posted. + */ + private function postAgentTalkFileMessage(\OCA\Talk\Room $room, Session $session, bool $threaded, int $fileId, bool $asVoiceMessage): void { + if (!class_exists('OCA\Talk\Service\ConversationFolderService')) { + return; + } + try { + $node = $this->taskProcessingService->getOutputFile($fileId); + } catch (\Throwable $e) { + $this->logger->warning('Could not resolve the generated file of an agent answer', [ + 'fileId' => $fileId, + 'exception' => $e, + ]); + return; + } + + // the stored mimetype of task output files is unreliable + // (providers write extension-less names), sniff the real one + try { + $head = fread($node->fopen('rb'), 4096); + $sniffedMimeType = is_string($head) ? (new \finfo(FILEINFO_MIME_TYPE))->buffer($head) : false; + } catch (\Throwable $e) { + $this->logger->warning('Could not read the generated file of an agent answer', [ + 'fileId' => $fileId, + 'exception' => $e, + ]); + return; + } + /** @var string $mimeType */ + $mimeType = is_string($sniffedMimeType) && $sniffedMimeType !== '' ? $sniffedMimeType : 'application/octet-stream'; + + try { + /** @var \OCA\Talk\Service\ConversationFolderService $conversationFolderService */ + $conversationFolderService = Server::get(\OCA\Talk\Service\ConversationFolderService::class); + $subfolder = $conversationFolderService->getOrCreateSubfolder($session->getUserId(), $room, false); + + $extension = match ($mimeType) { + 'image/png' => 'png', + 'image/jpeg' => 'jpg', + 'image/webp' => 'webp', + 'image/gif' => 'gif', + 'audio/mpeg' => 'mp3', + 'audio/wav', 'audio/x-wav' => 'wav', + 'application/pdf' => 'pdf', + default => preg_replace('/[^A-Za-z0-9]/', '', substr($mimeType, strpos($mimeType, '/') + 1)) ?: 'bin', + }; + $baseName = $asVoiceMessage ? 'voice-message' : 'generated'; + $fileName = $baseName . '-' . time() . '-' . random_int(1000, 9999) . '.' . $extension; + + // a server-side copy, the task output file itself stays untouched: + // the chat message attachment and the task output endpoints reference it + /** @var \OCP\Files\Node $finalNode */ + $finalNode = $node->copy($subfolder->getPath() . '/' . $fileName); + } catch (\Throwable $e) { + $this->logger->error('Could not store the generated file of an agent answer in the Talk conversation folder', [ + 'agentUserId' => $session->getUserId(), + 'fileId' => $fileId, + 'exception' => $e, + ]); + return; + } + + $metaData = ['mimeType' => $mimeType]; + if ($asVoiceMessage && ($mimeType === 'audio/mpeg' || $mimeType === 'audio/wav')) { + // Talk only renders the voice message bubble for these mimetypes + $metaData['messageType'] = 'voice-message'; + } + + $threadId = \OCA\Talk\Model\Thread::THREAD_NONE; + if ($threaded && $session->getThreadId() !== null) { + $threadId = $session->getThreadId(); + } + + try { + /** @var \OCA\Talk\Chat\ChatManager $chatManager */ + $chatManager = Server::get(\OCA\Talk\Chat\ChatManager::class); + $chatManager->addSystemMessage( + $room, + null, + 'users', + $session->getUserId(), + json_encode([ + 'message' => 'file_shared', + 'parameters' => [ + 'fileId' => (string)$finalNode->getId(), + 'metaData' => $metaData, + ], + ]), + new DateTime(), + true, + null, + null, + false, + false, + $threadId, + ); + } catch (\Throwable $e) { + $this->logger->error('Failed to post the generated file of an agent answer in Talk', [ + 'agentUserId' => $session->getUserId(), + 'fileId' => $fileId, + 'exception' => $e, + ]); + } + } + + /** + * When the answer was requested by an @mention from a group Talk room, + * post a short pointer message there, as the agent, in reply to the mention: + * it mentions the human user and links to the answer posted + * in the private 1-1 conversation with the agent. + * + * Messages sent in the 1-1 conversation or in its threads do not trigger + * a pointer: their answers are posted in that same conversation. + */ + private function postGroupRoomPointer(Session $session, \OCA\Talk\Room $oneToOneRoom, IComment $answerComment): void { + $seenRoomToken = $session->getAgentSeenRoomToken(); + $seenCommentId = $session->getAgentSeenCommentId(); + if ($seenRoomToken === null || $seenCommentId === null) { + return; + } + try { + $seenRoom = $this->getRoomByToken($seenRoomToken); + } catch (\OCA\Talk\Exceptions\RoomNotFoundException $e) { + $this->logger->warning('Cannot post the answer pointer message, the Talk room of the mention was not found', ['token' => $seenRoomToken]); + return; + } + if ($seenRoom->getType() === \OCA\Talk\Room::TYPE_ONE_TO_ONE) { + // the message was sent in the 1-1 conversation or one of its threads, + // the answer was posted there: no pointer message needed + return; + } + $requesterUserId = $session->getRequesterUserId(); + if ($requesterUserId === null) { + return; + } + + // the mention message to reply to must be a chat message of the handled room + try { + /** @var \OCA\Talk\Chat\CommentsManager $commentsManager */ + $commentsManager = Server::get(\OCA\Talk\Chat\CommentsManager::class); + $mentionComment = $commentsManager->getCommentsById([(string)$seenCommentId])[(string)$seenCommentId] ?? null; + } catch (\Throwable $e) { + $this->logger->warning('Cannot post the answer pointer message, the handled Talk message was not found', ['commentId' => $seenCommentId]); + return; + } + if ($mentionComment === null) { + return; + } + if ($mentionComment->getActorId() !== $requesterUserId) { + return; + } + + $threadId = (int)($answerComment->getMetaData()[\OCA\Talk\Model\Message::METADATA_THREAD_ID] ?? 0); + $this->postRoomPointer($seenRoom, $mentionComment, $oneToOneRoom, $answerComment, $session->getUserId(), $threadId !== 0 ? $threadId : null); + } + + /** + * Post a pointer message in the origin Talk room, as the agent, in reply to + * the human's message: it mentions the human user and links, inside the + * sentence, to the message posted in the private 1-1 conversation with + * the agent. + * + * The message is identical whether the agent answered or refused access, + * so that room members cannot infer who has access to the agent. + */ + private function postRoomPointer( + \OCA\Talk\Room $originRoom, + IComment $mentionComment, + \OCA\Talk\Room $oneToOneRoom, + IComment $answerComment, + string $agentUserId, + ?int $threadId = null, + ): void { + if ($mentionComment->getObjectType() !== 'chat' + || $mentionComment->getObjectId() !== (string)$originRoom->getId() + || $mentionComment->getActorType() !== 'users') { + return; + } + + $urlParams = [ + 'token' => $oneToOneRoom->getToken(), + '_fragment' => 'message_' . $answerComment->getId(), + ]; + if ($threadId !== null) { + // like Talk's notification links: thread context is needed to open the message + $urlParams['threadId'] = $threadId; + } + $link = $this->urlGenerator->linkToRouteAbsolute('spreed.Page.showCall', $urlParams); + $content = '@"' . $mentionComment->getActorId() . '" ' + . $this->l10n->t('I answered to you in [our private conversation](%s)', [$link]); + + /** @var \OCA\Talk\Chat\ChatManager $chatManager */ + $chatManager = Server::get(\OCA\Talk\Chat\ChatManager::class); + $chatManager->sendMessage( + $originRoom, + null, + 'users', + $agentUserId, + $content, + new DateTime(), + replyTo: $mentionComment, + ); + } + + /** + * Called by a listener when a human user reacts in Talk to a confirmation message + * (sensitive actions of the agency feature). + * + * The agent removes its own 👍/👎 (only the user's reaction remains), + * marks the message with 👀 as seen, and does the same as when confirming + * or rejecting from the assistant UI. + * + * @param string $humanUserId + * @param int $sessionId + * @param int $messageId The id of the Talk message that was reacted to + * @param string $reaction + * @return void + */ + public function handleConfirmationReaction(string $humanUserId, int $sessionId, int $messageId, string $reaction, \OCA\Talk\Room $room): void { + if (!in_array($reaction, [self::REACTION_APPROVE, self::REACTION_REJECT], true)) { + return; + } + try { + $session = $this->sessionMapper->getSessionById($sessionId); + } catch (DoesNotExistException|Exception $e) { + $this->logger->warning('Could not find the session matching the Talk reaction', [ + 'humanUserId' => $humanUserId, + 'sessionId' => $sessionId, + ]); + return; + } + if ($session->getRequesterUserId() === null) { + // the session is not an agent session + return; + } + if ($session->getRequesterUserId() !== $humanUserId) { + // the reaction was not added by the human user the agent session is dedicated to + return; + } + + // remove the agent's own thumbs so only the one of the user remains, + // then mark the confirmation as being processed + $this->deleteAgentReaction($room, $session->getUserId(), $messageId, self::REACTION_APPROVE); + $this->deleteAgentReaction($room, $session->getUserId(), $messageId, self::REACTION_REJECT); + $this->addAgentReaction($room, $session->getUserId(), $messageId, self::REACTION_READ); + // remember the seen message to later replace 👀 by ✅/🔴 after the confirmation run + $session->setAgentSeenRoomToken($room->getToken()); + $session->setAgentSeenCommentId($messageId); + try { + $this->sessionMapper->update($session); + } catch (Exception $e) { + $this->logger->error('Failed to store the seen confirmation of the agent session', ['exception' => $e]); + } + try { + $this->chatService->scheduleMessageGeneration( + $session->getUserId(), + $sessionId, + $reaction === self::REACTION_APPROVE ? 1 : 0, + $this->getOriginRoomInfo($session), + ); + } catch (InternalException|NotFoundException|BadRequestException|UnauthorizedException $e) { + $this->logger->error('Failed to schedule the confirmation follow-up after a Talk reaction', ['exception' => $e]); + } + } + + private function postAgentMessage( + \OCA\Talk\Room $room, + Session $session, + string $content, + ?string $referenceId = null, + bool $addApproveRejectReactions = false, + bool $threaded = true, + ): ?IComment { + if (trim($content) === '') { + return null; + } + $agentUser = $this->userManager->get($session->getUserId()); + if ($agentUser === null) { + return null; + } + /** @var \OCA\Talk\Chat\ChatManager $chatManager */ + $chatManager = Server::get(\OCA\Talk\Chat\ChatManager::class); + + $threadId = \OCA\Talk\Model\Thread::THREAD_NONE; + $threadTitle = ''; + if ($threaded) { + if ($session->getThreadId() !== null) { + // answer in the existing thread of the session + $threadId = $session->getThreadId(); + } else { + // create the thread of the "real chat session" + $threadId = \OCA\Talk\Model\Thread::THREAD_CREATE; + $threadTitle = $session->getTitle() ?? $this->getAgentDisplayName($this->getAgentByUserIdOrFail($session)); + } + } + + try { + $comment = $chatManager->sendMessage( + $room, + null, + 'users', + $session->getUserId(), + $content, + new DateTime(), + replyTo: null, + referenceId: $referenceId ?? '', + threadId: $threadId, + threadTitle: $threadTitle, + ); + } catch (\Throwable $e) { + $this->logger->error('Failed to post an agent message in Talk', ['exception' => $e]); + return null; + } + + if ($threadId === \OCA\Talk\Model\Thread::THREAD_CREATE) { + // ChatManager only sets the thread metadata, the thread and the thread_created + // system message are created by the caller, like Talk's ChatController does + /** @var \OCA\Talk\Service\ThreadService $threadService */ + $threadService = Server::get(\OCA\Talk\Service\ThreadService::class); + try { + $thread = $threadService->createThread($room, (int)$comment->getId(), $threadTitle); + } catch (\Throwable $e) { + $this->logger->error('Failed to create the Talk thread of the agent session', ['exception' => $e]); + return null; + } + try { + $chatManager->addSystemMessage( + $room, + null, + 'users', + $session->getUserId(), + json_encode([ + 'message' => 'thread_created', + 'parameters' => [ + 'thread' => (int)$comment->getId(), + 'title' => $thread->getName(), + ], + ]), + new DateTime(), + sendNotifications: false, + replyTo: $comment, + shouldSkipLastMessageUpdate: true, + ); + } catch (\Throwable $e) { + $this->logger->warning('Failed to post the thread_created system message', ['exception' => $e]); + } + } + + if ($threaded) { + // remember the thread, so the next messages are posted in it + $threadId = (int)($comment->getTopmostParentId() ?: $comment->getId()); + if ($session->getThreadId() !== $threadId) { + $session->setThreadId($threadId); + try { + $this->sessionMapper->update($session); + } catch (Exception $e) { + $this->logger->error('Failed to store the Talk thread of the agent session', ['exception' => $e]); + } + } + } + + if ($addApproveRejectReactions) { + $this->addAgentReaction($room, $session->getUserId(), (int)$comment->getId(), self::REACTION_APPROVE); + $this->addAgentReaction($room, $session->getUserId(), (int)$comment->getId(), self::REACTION_REJECT); + } + return $comment; + } + + /** + * Replace the 👀 reaction of the agent on the last seen human message + * by the outcome emoji (✅ answer generated, 🔴 generation failed). + */ + private function updateSeenReaction(Session $session, string $reaction): void { + $seenRoomToken = $session->getAgentSeenRoomToken(); + $seenCommentId = $session->getAgentSeenCommentId(); + if ($seenRoomToken === null || $seenCommentId === null) { + return; + } + try { + $room = $this->getRoomByToken($seenRoomToken); + } catch (\OCA\Talk\Exceptions\RoomNotFoundException $e) { + $this->logger->warning('Cannot update the reaction, the Talk room of the seen message was not found', ['token' => $seenRoomToken]); + return; + } + $this->deleteAgentReaction($room, $session->getUserId(), $seenCommentId, self::REACTION_READ); + $this->addAgentReaction($room, $session->getUserId(), $seenCommentId, $reaction); + // forget the seen message + $session->setAgentSeenRoomToken(null); + $session->setAgentSeenCommentId(null); + try { + $this->sessionMapper->update($session); + } catch (Exception $e) { + $this->logger->error('Failed to forget the seen message of the agent session', ['exception' => $e]); + } + } + + /** + * Called by ChattyLLMTaskListener once the answer of an agent session was generated + * + * @param Session $session + * @param bool $postedInTalk Whether the answer was successfully posted in Talk + * @return void + */ + public function markGenerationCompleted(Session $session, bool $postedInTalk): void { + if ($session->getUserId() === null) { + return; + } + if ($postedInTalk) { + $this->updateSeenReaction($session, self::REACTION_DONE); + } else { + $this->updateSeenReaction($session, self::REACTION_ERROR); + } + } + + /** + * Called by ChattyLLMTaskListener when the generation task of an agent session failed + * + * @param string $sessionOwnerUserId The agent user id owning the session + * @param int $sessionId + * @return void + */ + public function markGenerationFailed(string $sessionOwnerUserId, int $sessionId): void { + try { + $session = $this->sessionMapper->getUserSession($sessionOwnerUserId, $sessionId); + } catch (DoesNotExistException|Exception $e) { + $this->logger->warning('Cannot find the session of a failed agent message generation', ['sessionId' => $sessionId]); + return; + } + if ($session->getRequesterUserId() === null) { + return; + } + $this->updateSeenReaction($session, self::REACTION_ERROR); + } + + /** + * A pending agency confirmation was implicitly answered + * (a new human message was sent, or a new confirmation supersedes it): + * remove the agent's own 👍/👎 reactions on the pending confirmation + * message and clear the confirmation tracking. + * + * The session caller is responsible for persisting the cleared fields. + */ + private function removePendingConfirmationReactions(Session $session): void { + $commentId = $session->getAgencyConfirmationCommentId(); + $roomToken = $session->getAgencyConfirmationRoomToken(); + if ($commentId === null || $roomToken === null) { + return; + } + $session->setAgencyConfirmationRoomToken(null); + $session->setAgencyConfirmationCommentId(null); + try { + $room = $this->getRoomByToken($roomToken); + } catch (\OCA\Talk\Exceptions\RoomNotFoundException) { + // the room was deleted, the reactions are gone with it + return; + } + $this->deleteAgentReaction($room, $session->getUserId(), $commentId, self::REACTION_APPROVE); + $this->deleteAgentReaction($room, $session->getUserId(), $commentId, self::REACTION_REJECT); + } + + private function deleteAgentReaction(\OCA\Talk\Room $room, string $agentUserId, int $messageId, string $reaction): void { + if (trim($reaction) === '' || $this->userManager->get($agentUserId) === null) { + return; + } + try { + /** @var \OCA\Talk\Chat\ReactionManager $reactionManager */ + $reactionManager = Server::get(\OCA\Talk\Chat\ReactionManager::class); + $reactionManager->deleteReactionMessage( + $room, + 'users', + $agentUserId, + $agentUserId, + $messageId, + $reaction, + ); + } catch (\Throwable $e) { + $this->logger->warning('Failed to remove an automatic reaction on an agent message in Talk', ['exception' => $e]); + } + } + + private function addAgentReaction(\OCA\Talk\Room $room, string $agentUserId, int $messageId, string $reaction): void { + $agentUser = $this->userManager->get($agentUserId); + if ($agentUser === null) { + return; + } + try { + /** @var \OCA\Talk\Chat\ReactionManager $reactionManager */ + $reactionManager = Server::get(\OCA\Talk\Chat\ReactionManager::class); + $reactionManager->addReactionMessage( + $room, + 'users', + $agentUserId, + $agentUser->getDisplayName() ?: $agentUserId, + $messageId, + $reaction, + ); + } catch (\Throwable $e) { + $this->logger->error('Failed to add an automatic reaction on an agent message in Talk', ['exception' => $e]); + } + } + + private function getRoomByToken(string $token): \OCA\Talk\Room { + /** @var \OCA\Talk\Manager $talkManager */ + $talkManager = Server::get(\OCA\Talk\Manager::class); + return $talkManager->getRoomByToken($token); + } + + /** + * The name of the Talk room where the conversation of an agent session + * started, when the origin room is a group room. The identity context of + * the generation tasks mentions it, no other task attribute does. + * + * @param Session $session + * @return string|null + */ + private function getOriginRoomInfo(Session $session): ?string { + if ($session->getRoomToken() === null) { + return null; + } + try { + $originRoom = $this->getRoomByToken($session->getRoomToken()); + } catch (\OCA\Talk\Exceptions\RoomNotFoundException) { + return null; + } + if ($originRoom->getType() === \OCA\Talk\Room::TYPE_ONE_TO_ONE) { + // 1-1 conversations need no origin mention, everything happens there + return null; + } + return $originRoom->getName(); + } + + private function getAgentByUserIdOrFail(Session $session): Agent { + $agent = $this->agentsService->getAgentByUserId($session->getUserId()); + if ($agent === null) { + throw new AgentUserException('Agent user ' . $session->getUserId() . ' not found'); + } + return $agent; + } + +} diff --git a/lib/Service/AgentUserException.php b/lib/Service/AgentUserException.php new file mode 100644 index 00000000..d9b0d84c --- /dev/null +++ b/lib/Service/AgentUserException.php @@ -0,0 +1,13 @@ +userManager->userExists($userId)) { + throw new AgentUserException('The user id "' . $userId . '" already exists. Please choose another one.'); + } + + $password = bin2hex(random_bytes(self::PASSWORD_LENGTH)); + try { + $user = $this->userManager->createUser($userId, $password); + } catch (\Exception $e) { + throw new AgentUserException('Failed to create the agent user "' . $userId . '": ' . $e->getMessage(), previous: $e); + } + $user->setDisplayName($name); + return $user; + } +} diff --git a/lib/Service/AgentsService.php b/lib/Service/AgentsService.php new file mode 100644 index 00000000..ac9b3276 --- /dev/null +++ b/lib/Service/AgentsService.php @@ -0,0 +1,232 @@ + + * @throws Exception + */ + public function getAllAgents(): array { + return $this->agentMapper->getAllAgents(); + } + + /** + * @param int $id + * @return Agent|null + */ + public function getAgentById(int $id): ?Agent { + try { + return $this->agentMapper->getById($id); + } catch (DoesNotExistException|MultipleObjectsReturnedException|Exception $e) { + return null; + } + } + + /** + * @param string $userId + * @return Agent|null + */ + public function getAgentByUserId(string $userId): ?Agent { + try { + return $this->agentMapper->getByUserId($userId); + } catch (DoesNotExistException|MultipleObjectsReturnedException|Exception $e) { + return null; + } + } + + /** + * @param string|null $name + * @return array + * @throws Exception + */ + public function getAgentsByUserIds(array $userIds): array { + return $this->agentMapper->getByUserIds(array_values(array_filter($userIds))); + } + + /** + * @param string $name + * @param string $userId + * @param string $personality + * @param string $scope + * @param list $accessUsers User ids that may access the agent + * @param list $accessGroups Group ids that may access the agent + * @return Agent + * @throws AgentUserException + * @throws Exception + */ + public function createAgent( + string $name, string $userId, string $personality, string $scope, + array $accessUsers = [], array $accessGroups = [], + ): Agent { + $agent = new Agent(); + $agent->setUserId(trim(mb_strtolower($userId))); + $agent->setPersonality(trim($personality)); + $agent->setScope(trim($scope)); + try { + $this->agentUserService->createAgentUser($name, $userId); + } catch (AgentUserException $e) { + $this->logger->warning('Failed to provision the agent user', ['exception' => $e]); + throw $e; + } + $agent = $this->agentMapper->insert($agent); + if ($accessUsers !== [] || $accessGroups !== []) { + $this->agentMapper->setAcl((int)$agent->getId(), $this->makeAclEntries($accessUsers, $accessGroups)); + } + return $agent; + } + + /** + * @param int $id + * @param string|null $personality + * @param string|null $scope + * @param list|null $accessUsers When not null, replaces the list of user ids that may access the agent + * @param list|null $accessGroups When not null, replaces the list of group ids that may access the agent + * @return Agent + * @throws AgentUserException when the agent does not exist anymore + * @throws Exception + */ + public function updateAgent( + int $id, ?string $personality = null, ?string $scope = null, + ?array $accessUsers = null, ?array $accessGroups = null, + ): Agent { + try { + $agent = $this->agentMapper->getById($id); + } catch (DoesNotExistException $e) { + throw new AgentUserException('Agent not found'); + } + if ($personality !== null) { + if (strlen($personality) > self::MAX_PERSONALITY_LENGTH) { + throw new AgentUserException('The agent personality is too long'); + } + $agent->setPersonality(trim($personality)); + } + if ($scope !== null) { + if (strlen($scope) > self::MAX_SCOPE_LENGTH) { + throw new AgentUserException('The agent scope is too long'); + } + $agent->setScope(trim($scope)); + } + $agent = $this->agentMapper->update($agent); + if ($accessUsers !== null || $accessGroups !== null) { + $entries = $this->getAgentAccess($id); + $accessUsers ??= $entries['users']; + $accessGroups ??= $entries['groups']; + $this->agentMapper->setAcl($id, $this->makeAclEntries($accessUsers, $accessGroups)); + } + return $agent; + } + + /** + * @param int $id + * @param bool $deleteUser Whether the related NC user should be deleted too + * @return void + * @throws Exception + */ + public function deleteAgent(int $id, bool $deleteUser = false): void { + try { + $agent = $this->agentMapper->getById($id); + } catch (DoesNotExistException $e) { + return; + } + $this->agentMapper->deleteAgent($agent); + + if ($deleteUser) { + $user = $this->userManager->get($agent->getUserId()); + if ($user !== null) { + $user->delete(); + } + } + } + + /** + * The user ids and group ids that may access the agent. + * Both lists are empty when the access is not restricted. + * + * @param int $agentId + * @return array{users: list, groups: list} + * @throws Exception + */ + public function getAgentAccess(int $agentId): array { + $users = []; + $groups = []; + foreach ($this->agentMapper->getAcl($agentId) as $entry) { + if ($entry['type'] === self::ACL_TYPE_USER) { + $users[] = $entry['target']; + } elseif ($entry['type'] === self::ACL_TYPE_GROUP) { + $groups[] = $entry['target']; + } + } + return ['users' => $users, 'groups' => $groups]; + } + + /** + * Whether the user may access the agent: + * an empty access list means every user has access, + * otherwise the user must be listed by id or belong to a listed group. + * + * @param Agent $agent + * @param IUser $user + * @return bool + * @throws Exception + */ + public function canUserAccessAgent(Agent $agent, IUser $user): bool { + return $this->agentMapper->canUserAccess( + (int)$agent->getId(), + $user->getUID(), + $this->groupManager->getUserGroupIds($user), + ); + } + + /** + * @param list $accessUsers + * @param list $accessGroups + * @return list + */ + private function makeAclEntries(array $accessUsers, array $accessGroups): array { + $entries = []; + foreach ($accessUsers as $userId) { + $entries[] = ['type' => self::ACL_TYPE_USER, 'target' => $userId]; + } + foreach ($accessGroups as $groupId) { + $entries[] = ['type' => self::ACL_TYPE_GROUP, 'target' => $groupId]; + } + return $entries; + } +} diff --git a/lib/Service/ChatService.php b/lib/Service/ChatService.php index 4abeca26..ff7de696 100644 --- a/lib/Service/ChatService.php +++ b/lib/Service/ChatService.php @@ -51,7 +51,7 @@ public function __construct( * @throws InternalException * @throws UnauthorizedException */ - public function createChatSession(?string $userId, ?int $timestamp = null, ?string $title = null): Session { + public function createChatSession(?string $userId, ?int $timestamp = null, ?string $title = null, ?string $systemPrompt = null, ?string $requesterUserId = null): Session { if ($userId === null) { throw new UnauthorizedException($this->l10n->t('Unauthorized')); } @@ -59,6 +59,10 @@ public function createChatSession(?string $userId, ?int $timestamp = null, ?stri if ($user === null) { throw new UnauthorizedException($this->l10n->t('User not found')); } + // agent sessions are owned by the agent user account, + // the user-related placeholders address the human requester + $placeholderUserId = $requesterUserId ?? $userId; + $placeholderUser = $placeholderUserId === $userId ? $user : $this->userManager->get($placeholderUserId); $timestamp ??= $this->timeFactory->now()->getTimestamp(); @@ -66,13 +70,16 @@ public function createChatSession(?string $userId, ?int $timestamp = null, ?stri $timestamp = intdiv($timestamp, 1000); } - $userInstructions = $this->appConfig->getValueString( - Application::APP_ID, - 'chat_user_instructions', - Application::CHAT_USER_INSTRUCTIONS, - lazy: true, - ) ?: Application::CHAT_USER_INSTRUCTIONS; - $userInstructions = str_replace('{user}', $user->getDisplayName(), $userInstructions); + $userInstructions = $systemPrompt; + if ($userInstructions === null) { + $userInstructions = $this->appConfig->getValueString( + Application::APP_ID, + 'chat_user_instructions', + Application::CHAT_USER_INSTRUCTIONS, + lazy: true, + ) ?: Application::CHAT_USER_INSTRUCTIONS; + } + $userInstructions = str_replace('{user}', $placeholderUser?->getDisplayName() ?: $placeholderUserId, $userInstructions); // the user's personal instructions come after the admin's and must not override them $personalInstructionsEnabled = $this->appConfig->getValueString(Application::APP_ID, 'chat_personal_instructions_enabled', '0', lazy: true) === '1'; @@ -89,6 +96,7 @@ public function createChatSession(?string $userId, ?int $timestamp = null, ?stri $session->setTimestamp($timestamp); $session->setAgencyConversationToken(null); $session->setAgencyPendingActions(null); + $session->setRequesterUserId($requesterUserId); try { $this->sessionMapper->insert($session); } catch (Exception $e) { @@ -411,20 +419,23 @@ public function deleteSessionMessage(?string $userId, int $sessionId, int $messa * @throws NotFoundException * @throws UnauthorizedException */ - public function scheduleMessageGeneration(?string $userId, int $sessionId, int $agencyConfirm = 0): int { + public function scheduleMessageGeneration(?string $userId, int $sessionId, int $agencyConfirm = 0, ?string $originRoomInfo = null): int { if ($userId === null) { throw new UnauthorizedException($this->l10n->t('Unauthorized')); } try { - $sessionExists = $this->sessionMapper->exists($userId, $sessionId); - } catch (Exception $e) { + $session = $this->sessionMapper->getUserSession($userId, $sessionId); + } catch (DoesNotExistException $e) { + throw new NotFoundException($this->l10n->t('Session not found'), previous: $e); + } catch (MultipleObjectsReturnedException|Exception $e) { throw new InternalException(previous: $e); } - if (!$sessionExists) { - throw new NotFoundException($this->l10n->t('Session not found')); - } + // agent sessions belong to the agent user account: the memories to inject + // come from the human requester, not from the agent + $memoriesUserId = $session->getRequesterUserId() ?? $userId; if ($this->isContextAgentAvailable()) { + $identityLine = $this->getIdentityLine($session, $originRoomInfo); try { $lastUserMessage = $this->messageMapper->getLastHumanMessage($sessionId); } catch (DoesNotExistException $e) { @@ -433,13 +444,6 @@ public function scheduleMessageGeneration(?string $userId, int $sessionId, int $ throw new InternalException(previous: $e); } - try { - $session = $this->sessionMapper->getUserSession($userId, $sessionId); - } catch (DoesNotExistException $e) { - throw new NotFoundException($this->l10n->t('Session not found'), previous: $e); - } catch (MultipleObjectsReturnedException|Exception $e) { - throw new InternalException(previous: $e); - } $lastConversationToken = $session->getAgencyConversationToken() ?? self::EMPTY_CONVERSATION_TOKEN; $lastAttachments = $lastUserMessage->jsonSerialize()['attachments']; @@ -452,22 +456,26 @@ public function scheduleMessageGeneration(?string $userId, int $sessionId, int $ ) { // audio agency $fileId = $audioAttachment['file_id']; - $taskId = $this->scheduleAgencyAudioTask($userId, $fileId, $agencyConfirm, $lastConversationToken, $sessionId, $lastUserMessage->getId()); + $taskId = $this->scheduleAgencyAudioTask($userId, $fileId, $agencyConfirm, $lastConversationToken, $sessionId, $lastUserMessage->getId(), $memoriesUserId, $identityLine); } elseif ($this->isMultimodalContextAgentAvailable()) { // multimodal agency $prompt = $lastUserMessage->getContent(); $inputAttachments = array_map(static function (array $attachment) { return $attachment['file_id']; }, $lastAttachments); - $taskId = $this->scheduleAgencyMultimodalTask($userId, $prompt, $agencyConfirm, $lastConversationToken, $sessionId, $inputAttachments); + $taskId = $this->scheduleAgencyMultimodalTask($userId, $prompt, $agencyConfirm, $lastConversationToken, $sessionId, $inputAttachments, $memoriesUserId, $identityLine); } else { // classic agency $prompt = $lastUserMessage->getContent(); - $taskId = $this->scheduleAgencyTask($userId, $prompt, $agencyConfirm, $lastConversationToken, $sessionId); + $taskId = $this->scheduleAgencyTask($userId, $prompt, $agencyConfirm, $lastConversationToken, $sessionId, memoriesUserId: $memoriesUserId, identityLine: $identityLine); } } else { // classic chat + $identityLine = $this->getIdentityLine($session, $originRoomInfo); $systemPrompt = $this->getSessionSystemPrompt($sessionId); + if ($identityLine !== null) { + $systemPrompt = $identityLine . "\n\n" . $systemPrompt; + } try { $history = $this->getRawLastMessages($sessionId); } catch (Exception|AppConfigTypeConflictException $e) { @@ -495,7 +503,7 @@ public function scheduleMessageGeneration(?string $userId, int $sessionId, int $ // for an audio chat task, let's try to get the remote audio IDs for all the previous audio messages $history = $this->getAudioHistory($history); $fileId = $audioAttachment['file_id']; - $taskId = $this->scheduleAudioChatTask($userId, $fileId, $systemPrompt, $history, $sessionId, $lastUserMessage->getId()); + $taskId = $this->scheduleAudioChatTask($userId, $fileId, $systemPrompt, $history, $sessionId, $lastUserMessage->getId(), $memoriesUserId); } else { if ($this->isMultimodalChatAvailable()) { // for a multimodal chat also attachments need to be added to the history @@ -520,7 +528,7 @@ public function scheduleMessageGeneration(?string $userId, int $sessionId, int $ $lastAttachments = array_map(static function (array $attachment) { return $attachment['file_id']; }, $lastAttachments); - $taskId = $this->scheduleMultimodalChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $historyMessages, $sessionId, $lastAttachments); + $taskId = $this->scheduleMultimodalChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $historyMessages, $sessionId, $lastAttachments, $memoriesUserId); } else { // for a text chat task only use text in the history $historyMessages = array_map(static function (Message $message) { @@ -529,7 +537,7 @@ public function scheduleMessageGeneration(?string $userId, int $sessionId, int $ 'content' => $message->getContent(), ]); }, $history); - $taskId = $this->scheduleLLMChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $historyMessages, $sessionId); + $taskId = $this->scheduleLLMChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $historyMessages, $sessionId, memoriesUserId: $memoriesUserId); } } } @@ -677,20 +685,55 @@ private function getSessionSystemPrompt(int $sessionId): string { return ''; } + /** + * Agent sessions are owned by the agent user account: without an explicit + * statement, the model neither knows which agent it is, nor which human + * user it is talking with. This describes both identities. + * + * @param Session $session + * @param string|null $originRoomInfo The name of the Talk room where the + * conversation started, when it is not + * a 1-1 conversation + * @return string|null Null for sessions that are not agent sessions + */ + private function getIdentityLine(Session $session, ?string $originRoomInfo): ?string { + $requesterUserId = $session->getRequesterUserId(); + if ($requesterUserId === null) { + return null; + } + $agentUserId = $session->getUserId(); + $agentDisplayName = $this->userManager->get($agentUserId)?->getDisplayName() ?: $agentUserId; + $requesterDisplayName = $this->userManager->get($requesterUserId)?->getDisplayName() ?: $requesterUserId; + $identityLine = $this->l10n->t('You, the assistant, are "%1$s" (account "@%2$s").', [$agentDisplayName, $agentUserId]) + . ' ' . $this->l10n->t('You are assisting %1$s (account "@%2$s") through Talk.', [$requesterDisplayName, $requesterUserId]) + . ' ' . $this->l10n->t('When asked who you are talking with, e.g. "who am I" or "who are you talking to", refer to the human identity. When asked "who are you", refer to your own assistant identity.'); + if ($originRoomInfo !== null) { + $identityLine .= ' ' . $this->l10n->t('The conversation started in the Talk room "%s".', [$originRoomInfo]); + } + return $identityLine; + } + /** * Pass the session's user instructions to an agency task if its provider supports them. * The agent appends them to its own system prompt, they do not replace it. * If they cannot be read, the task is scheduled without them. + * The identity line is passed by agent sessions, so the model knows + * which assistant agent it is and which human user it is talking with. */ - private function addAgencyUserInstructions(array &$taskInput, array $optionalInputShape, int $sessionId): void { + private function addAgencyUserInstructions(array &$taskInput, array $optionalInputShape, ?string $identityLine, int $sessionId): void { if (!isset($optionalInputShape['user_instructions'])) { return; } + $userInstructions = ''; try { $userInstructions = $this->getSessionSystemPrompt($sessionId); } catch (InternalException $e) { $this->logger->warning('Could not get the user instructions of the session, the agent will run without them', ['exception' => $e, 'sessionId' => $sessionId]); - return; + } + if ($identityLine !== null) { + $userInstructions = $userInstructions === '' + ? $identityLine + : $identityLine . "\n\n" . $userInstructions; } if ($userInstructions !== '') { $taskInput['user_instructions'] = $userInstructions; @@ -781,6 +824,7 @@ private function scheduleLLMChatTask( array $history, int $sessionId, bool $isMessage = true, + ?string $memoriesUserId = null, ): int { $customId = ($isMessage ? 'chatty-llm:' @@ -792,7 +836,7 @@ private function scheduleLLMChatTask( 'history' => $history, ]; if (isset($this->taskProcessingManager->getAvailableTaskTypes()[TextToTextChat::ID]['optionalInputShape']['memories'])) { - $input['memories'] = $this->sessionSummaryService->getMemories($userId); + $input['memories'] = $this->sessionSummaryService->getMemories($memoriesUserId ?? $userId); } $task = new Task(TextToTextChat::ID, $input, Application::APP_ID . ':chatty-llm', $userId, $customId); /** @psalm-suppress UndefinedMethod */ @@ -825,6 +869,7 @@ private function scheduleMultimodalChatTask( array $history, int $sessionId, array $attachmentsHistory, + ?string $memoriesUserId = null, ): int { $customId = 'chatty-llm:' . $sessionId; $this->checkIfSessionIsThinking($userId, $customId); @@ -868,6 +913,8 @@ private function scheduleAgencyTask( string $conversationToken, int $sessionId, bool $withUserInstructions = true, + ?string $memoriesUserId = null, + ?string $identityLine = null, ): int { $customId = 'chatty-llm:' . $sessionId; $this->checkIfSessionIsThinking($userId, $customId); @@ -879,10 +926,10 @@ private function scheduleAgencyTask( /** @psalm-suppress UndefinedClass */ $optionalInputShape = $this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID]['optionalInputShape'] ?? []; if (isset($optionalInputShape['memories'])) { - $taskInput['memories'] = $this->sessionSummaryService->getMemories($userId); + $taskInput['memories'] = $this->sessionSummaryService->getMemories($memoriesUserId ?? $userId); } if ($withUserInstructions) { - $this->addAgencyUserInstructions($taskInput, $optionalInputShape, $sessionId); + $this->addAgencyUserInstructions($taskInput, $optionalInputShape, $identityLine, $sessionId); } /** @psalm-suppress UndefinedClass */ $task = new Task( @@ -923,6 +970,8 @@ private function scheduleAgencyMultimodalTask( string $conversationToken, int $sessionId, array $inputAttachments, + ?string $memoriesUserId = null, + ?string $identityLine = null, ): int { $customId = 'chatty-llm:' . $sessionId; $this->checkIfSessionIsThinking($userId, $customId); @@ -932,10 +981,10 @@ private function scheduleAgencyMultimodalTask( 'confirmation' => $confirmation, 'conversation_token' => $conversationToken, ]; - $taskInput['memories'] = $this->sessionSummaryService->getMemories($userId); + $taskInput['memories'] = $this->sessionSummaryService->getMemories($memoriesUserId ?? $userId); /** @psalm-suppress UndefinedClass */ $optionalInputShape = $this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID]['optionalInputShape'] ?? []; - $this->addAgencyUserInstructions($taskInput, $optionalInputShape, $sessionId); + $this->addAgencyUserInstructions($taskInput, $optionalInputShape, $identityLine, $sessionId); /** @psalm-suppress UndefinedClass */ $task = new Task( \OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID, @@ -973,6 +1022,7 @@ private function scheduleAudioChatTask( array $history, int $sessionId, int $queryMessageId, + ?string $memoriesUserId = null, ): int { $customId = 'chatty-llm:' . $sessionId . ':' . $queryMessageId; $this->checkIfSessionIsThinking($userId, $customId); @@ -983,7 +1033,7 @@ private function scheduleAudioChatTask( ]; /** @psalm-suppress UndefinedClass */ if (isset($this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\AudioToAudioChat::ID]['optionalInputShape']['memories'])) { - $input['memories'] = $this->sessionSummaryService->getMemories($userId); + $input['memories'] = $this->sessionSummaryService->getMemories($memoriesUserId ?? $userId); } /** @psalm-suppress UndefinedClass */ $task = new Task( @@ -1020,6 +1070,8 @@ private function scheduleAgencyAudioTask( string $conversationToken, int $sessionId, int $queryMessageId, + ?string $memoriesUserId = null, + ?string $identityLine = null, ): int { $customId = 'chatty-llm:' . $sessionId . ':' . $queryMessageId; $this->checkIfSessionIsThinking($userId, $customId); @@ -1031,9 +1083,9 @@ private function scheduleAgencyAudioTask( /** @psalm-suppress UndefinedClass */ $optionalInputShape = $this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID]['optionalInputShape'] ?? []; if (isset($optionalInputShape['memories'])) { - $taskInput['memories'] = $this->sessionSummaryService->getMemories($userId); + $taskInput['memories'] = $this->sessionSummaryService->getMemories($memoriesUserId ?? $userId); } - $this->addAgencyUserInstructions($taskInput, $optionalInputShape, $sessionId); + $this->addAgencyUserInstructions($taskInput, $optionalInputShape, $identityLine, $sessionId); /** @psalm-suppress UndefinedClass */ $task = new Task( \OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID, diff --git a/lib/Settings/Admin.php b/lib/Settings/Admin.php index e75625d5..13a1f0b1 100644 --- a/lib/Settings/Admin.php +++ b/lib/Settings/Admin.php @@ -9,15 +9,19 @@ use OCA\Assistant\AppInfo\Application; use OCA\Assistant\Service\AgentSkillsService; +use OCA\Assistant\Service\AgentsService; use OCA\Assistant\TaskProcessing\TextToStickerTaskType; use OCP\AppFramework\Http\TemplateResponse; use OCP\AppFramework\Services\IInitialState; +use OCP\DB\Exception; use OCP\IAppConfig; +use OCP\IUserManager; use OCP\Settings\ISettings; use OCP\TaskProcessing\IManager as ITaskProcessingManager; use OCP\TaskProcessing\TaskTypes\AudioToText; use OCP\TaskProcessing\TaskTypes\TextToImage; use OCP\TaskProcessing\TaskTypes\TextToText; +use Psr\Log\LoggerInterface; class Admin implements ISettings { @@ -26,6 +30,9 @@ public function __construct( private IInitialState $initialStateService, private ITaskProcessingManager $taskProcessingManager, private AgentSkillsService $agentSkillsService, + private AgentsService $agentsService, + private IUserManager $userManager, + private LoggerInterface $logger, ) { } @@ -64,6 +71,25 @@ public function getForm(): TemplateResponse { $globalSkillsConfig = $this->agentSkillsService->getGlobalSkillsConfig(); + $agents = []; + try { + foreach ($this->agentsService->getAllAgents() as $agent) { + $user = $this->userManager->get($agent->getUserId()); + $access = $this->agentsService->getAgentAccess((int)$agent->getId()); + $agents[] = [ + 'id' => (int)$agent->getId(), + 'user_id' => $agent->getUserId(), + 'display_name' => $user?->getDisplayName() ?: $agent->getUserId(), + 'personality' => $agent->getPersonality(), + 'scope' => $agent->getScope(), + 'access_users' => $access['users'], + 'access_groups' => $access['groups'], + ]; + } + } catch (Exception $e) { + $this->logger->warning('Could not load the agents', ['exception' => $e]); + } + $adminConfig = [ 'task_processing_available' => $taskProcessingAvailable, 'assistant_enabled' => $assistantEnabled, @@ -83,6 +109,7 @@ public function getForm(): TemplateResponse { 'context_agent_available' => $contextAgentAvailable, 'global_skills_admin_uid' => $globalSkillsConfig['admin_uid'], 'global_skills_path' => $globalSkillsConfig['path'], + 'agents' => $agents, ]; $this->initialStateService->provideInitialState('admin-config', $adminConfig); diff --git a/openapi.json b/openapi.json index 4d1d4a49..8aaeacad 100644 --- a/openapi.json +++ b/openapi.json @@ -3887,6 +3887,1206 @@ } } }, + "/ocs/v2.php/apps/assistant/api/{apiVersion}/agents": { + "get": { + "operationId": "agents_api-list-agents", + "summary": "List all agents (admin only)", + "description": "This endpoint requires admin access", + "tags": [ + "agents" + ], + "security": [ + { + "bearer_auth": [] + }, + { + "basic_auth": [] + } + ], + "parameters": [ + { + "name": "apiVersion", + "in": "path", + "required": true, + "schema": { + "type": "string", + "enum": [ + "v1" + ], + "default": "v1" + } + }, + { + "name": "OCS-APIRequest", + "in": "header", + "description": "Required to be true for the API request to pass", + "required": true, + "schema": { + "type": "boolean", + "default": true + } + } + ], + "responses": { + "200": { + "description": "Agents returned", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "agents" + ], + "properties": { + "agents": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "user_id", + "display_name", + "personality", + "scope", + "access_users", + "access_groups" + ], + "properties": { + "id": { + "type": "integer", + "format": "int64" + }, + "user_id": { + "type": "string" + }, + "display_name": { + "type": "string" + }, + "personality": { + "type": "string", + "nullable": true + }, + "scope": { + "type": "string", + "nullable": true + }, + "access_users": { + "type": "array", + "items": { + "type": "string" + } + }, + "access_groups": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + } + } + } + } + } + } + }, + "403": { + "description": "User is not an admin", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + }, + { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": {} + } + } + } + } + ] + } + } + } + }, + "401": { + "description": "Current user is not logged in", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": {} + } + } + } + } + } + } + } + } + }, + "post": { + "operationId": "agents_api-create-agent", + "summary": "Create an agent (admin only)", + "description": "A dedicated Nextcloud user is created in the default database backend if it does not exist yet.\nThis endpoint requires admin access", + "tags": [ + "agents" + ], + "security": [ + { + "bearer_auth": [] + }, + { + "basic_auth": [] + } + ], + "requestBody": { + "required": false, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "default": "", + "description": "The name of the agent" + }, + "userId": { + "type": "string", + "default": "", + "description": "The user id of the agent's dedicated Nextcloud user" + }, + "personality": { + "type": "string", + "default": "", + "description": "The agent's system prompt" + }, + "scope": { + "type": "string", + "default": "", + "description": "Optional scope for later use" + }, + "accessUsers": { + "type": "array", + "default": [], + "description": "User ids that may access the agent (empty = every user)", + "items": { + "type": "string" + } + }, + "accessGroups": { + "type": "array", + "default": [], + "description": "Group ids that may access the agent (empty = every user)", + "items": { + "type": "string" + } + } + } + } + } + } + }, + "parameters": [ + { + "name": "apiVersion", + "in": "path", + "required": true, + "schema": { + "type": "string", + "enum": [ + "v1" + ], + "default": "v1" + } + }, + { + "name": "OCS-APIRequest", + "in": "header", + "description": "Required to be true for the API request to pass", + "required": true, + "schema": { + "type": "boolean", + "default": true + } + } + ], + "responses": { + "201": { + "description": "Agent created", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "agent" + ], + "properties": { + "agent": { + "type": "object", + "required": [ + "id", + "user_id", + "display_name", + "personality", + "scope", + "access_users", + "access_groups" + ], + "properties": { + "id": { + "type": "integer", + "format": "int64" + }, + "user_id": { + "type": "string" + }, + "display_name": { + "type": "string" + }, + "personality": { + "type": "string", + "nullable": true + }, + "scope": { + "type": "string", + "nullable": true + }, + "access_users": { + "type": "array", + "items": { + "type": "string" + } + }, + "access_groups": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + } + } + } + } + } + }, + "400": { + "description": "Invalid data", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + } + }, + "403": { + "description": "User is not an admin", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + }, + { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": {} + } + } + } + } + ] + } + } + } + }, + "500": { + "description": "Agent creation failed", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + } + }, + "401": { + "description": "Current user is not logged in", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": {} + } + } + } + } + } + } + } + } + } + }, + "/ocs/v2.php/apps/assistant/api/{apiVersion}/agents/{id}": { + "put": { + "operationId": "agents_api-update-agent", + "summary": "Update an agent (admin only)", + "description": "This endpoint requires admin access", + "tags": [ + "agents" + ], + "security": [ + { + "bearer_auth": [] + }, + { + "basic_auth": [] + } + ], + "requestBody": { + "required": false, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "personality": { + "type": "string", + "nullable": true, + "default": null, + "description": "The new personality (system prompt)" + }, + "scope": { + "type": "string", + "nullable": true, + "default": null, + "description": "The new scope" + }, + "accessUsers": { + "type": "array", + "nullable": true, + "default": null, + "description": "When not null, replaces the user ids that may access the agent", + "items": { + "type": "string" + } + }, + "accessGroups": { + "type": "array", + "nullable": true, + "default": null, + "description": "When not null, replaces the group ids that may access the agent", + "items": { + "type": "string" + } + } + } + } + } + } + }, + "parameters": [ + { + "name": "apiVersion", + "in": "path", + "required": true, + "schema": { + "type": "string", + "enum": [ + "v1" + ], + "default": "v1" + } + }, + { + "name": "id", + "in": "path", + "description": "The agent id", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + }, + { + "name": "OCS-APIRequest", + "in": "header", + "description": "Required to be true for the API request to pass", + "required": true, + "schema": { + "type": "boolean", + "default": true + } + } + ], + "responses": { + "200": { + "description": "Agent updated", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "agent" + ], + "properties": { + "agent": { + "type": "object", + "required": [ + "id", + "user_id", + "display_name", + "personality", + "scope", + "access_users", + "access_groups" + ], + "properties": { + "id": { + "type": "integer", + "format": "int64" + }, + "user_id": { + "type": "string" + }, + "display_name": { + "type": "string" + }, + "personality": { + "type": "string", + "nullable": true + }, + "scope": { + "type": "string", + "nullable": true + }, + "access_users": { + "type": "array", + "items": { + "type": "string" + } + }, + "access_groups": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + } + } + } + } + } + }, + "400": { + "description": "Invalid data", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + } + }, + "403": { + "description": "User is not an admin", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + }, + { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": {} + } + } + } + } + ] + } + } + } + }, + "404": { + "description": "Agent not found", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + } + }, + "500": { + "description": "Agent update failed", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + } + }, + "401": { + "description": "Current user is not logged in", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": {} + } + } + } + } + } + } + } + } + }, + "delete": { + "operationId": "agents_api-delete-agent", + "summary": "Delete an agent (admin only)", + "description": "This endpoint requires admin access", + "tags": [ + "agents" + ], + "security": [ + { + "bearer_auth": [] + }, + { + "basic_auth": [] + } + ], + "parameters": [ + { + "name": "apiVersion", + "in": "path", + "required": true, + "schema": { + "type": "string", + "enum": [ + "v1" + ], + "default": "v1" + } + }, + { + "name": "id", + "in": "path", + "description": "The agent id", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + }, + { + "name": "deleteUser", + "in": "query", + "description": "Whether the related Nextcloud user should be deleted too", + "schema": { + "type": "integer", + "format": "int64", + "default": 0 + } + }, + { + "name": "OCS-APIRequest", + "in": "header", + "description": "Required to be true for the API request to pass", + "required": true, + "schema": { + "type": "boolean", + "default": true + } + } + ], + "responses": { + "200": { + "description": "Agent deleted", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "action" + ], + "properties": { + "action": { + "type": "string" + } + } + } + } + } + } + } + } + } + }, + "403": { + "description": "User is not an admin", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + }, + { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": {} + } + } + } + } + ] + } + } + } + }, + "404": { + "description": "Agent not found", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + } + }, + "500": { + "description": "Agent deletion failed", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + } + }, + "401": { + "description": "Current user is not logged in", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": {} + } + } + } + } + } + } + } + } + } + }, "/ocs/v2.php/apps/assistant/chat/sessions": { "post": { "operationId": "chattyllm-new-session-restful", @@ -7895,5 +9095,10 @@ } } }, - "tags": [] + "tags": [ + { + "name": "agents_api", + "description": "CRUD access to the agents of the \"agent as a user\" feature (admin only)" + } + ] } diff --git a/psalm.xml b/psalm.xml index 05be2e86..aa36b04f 100644 --- a/psalm.xml +++ b/psalm.xml @@ -38,6 +38,7 @@ + @@ -45,6 +46,7 @@ + diff --git a/src/components/AdminSettings.vue b/src/components/AdminSettings.vue index c0200eb1..bbd1f00b 100644 --- a/src/components/AdminSettings.vue +++ b/src/components/AdminSettings.vue @@ -240,6 +240,15 @@ {{ t('assistant', 'No global skills folder configured') }} +
+

+ {{ t('assistant', 'Talk agents') }} +

+ +
@@ -267,6 +276,8 @@ import { generateUrl } from '@nextcloud/router' import { delay } from '../utils.js' +import TalkAgentUsers from './TalkAgentUsers.vue' + export default { name: 'AdminSettings', @@ -282,6 +293,7 @@ export default { NcFormBox, NcFormBoxSwitch, OpenInNewIcon, + TalkAgentUsers, }, data() { @@ -297,6 +309,18 @@ export default { }, methods: { + onAgentCreated(agent) { + this.state.agents.push(agent) + }, + onAgentUpdated(agent) { + const index = this.state.agents.findIndex((a) => a.id === agent.id) + if (index !== -1) { + this.state.agents.splice(index, 1, agent) + } + }, + onAgentDeleted(agent) { + this.state.agents = this.state.agents.filter((a) => a.id !== agent.id) + }, isUnsignedIntStr(value) { return /^\d+$/.test(value) }, diff --git a/src/components/TalkAgentUsers.vue b/src/components/TalkAgentUsers.vue new file mode 100644 index 00000000..b4f57ee9 --- /dev/null +++ b/src/components/TalkAgentUsers.vue @@ -0,0 +1,486 @@ + + + + + + diff --git a/tests/unit/Listener/ChattyLLMTaskListenerTest.php b/tests/unit/Listener/ChattyLLMTaskListenerTest.php index 845fb614..63838bcf 100644 --- a/tests/unit/Listener/ChattyLLMTaskListenerTest.php +++ b/tests/unit/Listener/ChattyLLMTaskListenerTest.php @@ -14,11 +14,14 @@ use OCA\Assistant\Db\ChattyLLM\Session; use OCA\Assistant\Db\ChattyLLM\SessionMapper; use OCA\Assistant\Listener\ChattyLLMTaskListener; +use OCA\Assistant\Service\AgentTalkService; use OCA\Assistant\Service\NotificationService; use OCA\Assistant\Service\TaskProcessingService; use OCP\AppFramework\Db\DoesNotExistException; use OCP\TaskProcessing\Events\TaskSuccessfulEvent; use OCP\TaskProcessing\Task; +use OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction; +use OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction; use OCP\TaskProcessing\TaskTypes\TextToTextChat; use PHPUnit\Framework\MockObject\MockObject; use Psr\Log\LoggerInterface; @@ -47,10 +50,15 @@ protected function setUp(): void { $session->setAssignmentId(null); $this->sessionMapper->method('getUserSession')->willReturn($session); + $this->rebuildListener($this->createMock(AgentTalkService::class)); + } + + private function rebuildListener(AgentTalkService $agentTalkService, ?SessionMapper $sessionMapper = null): void { $this->listener = new ChattyLLMTaskListener( $this->messageMapper, - $this->sessionMapper, + $sessionMapper ?? $this->sessionMapper, $this->createMock(TaskProcessingService::class), + $agentTalkService, $this->createMock(LoggerInterface::class), $this->createMock(NotificationService::class), ); @@ -138,4 +146,144 @@ public function testTitleTasksAreUntouched(): void { $this->listener->handle($event); } + + private function makeAudioTask(array $input, array $output): Task { + $task = new Task( + ContextAgentAudioInteraction::ID, + $input, + 'assistant:chatty-llm', + 'admin', + 'chatty-llm:' . self::SESSION_ID, + ); + $task->setId(self::TASK_ID); + $task->setOutput($output); + return $task; + } + + private function givenUnstoredMessage(): void { + $this->messageMapper->method('getMessageByTaskId') + ->with(self::SESSION_ID, self::TASK_ID) + ->willThrowException(new DoesNotExistException('Message not found')); + $this->messageMapper->method('insert') + ->willReturnCallback(static function (Message $message): Message { + $message->setId(3); + return $message; + }); + } + + private function givenAgentSession(SessionMapper&MockObject $sessionMapper, Session $session): void { + $sessionMapper->method('getUserSession')->willReturn($session); + $sessionMapper->method('update'); + } + + public function testAgentReplyReceivesTheAudioTranscript(): void { + $sessionMapper = $this->createMock(SessionMapper::class); + $session = new Session(); + $session->setAssignmentId(null); + $session->setRequesterUserId('alice'); + $this->givenAgentSession($sessionMapper, $session); + + $this->givenUnstoredMessage(); + + /** @var AgentTalkService&MockObject $agentTalkService */ + $agentTalkService = $this->createMock(AgentTalkService::class); + $agentTalkService->expects($this->once()) + ->method('postAgentTalkReply') + ->with( + $session, + 'The agent answer', + null, + 'Hello agent, what do you hear?', + 456, + [], + ); + $this->rebuildListener($agentTalkService, $sessionMapper); + + $task = $this->makeAudioTask( + ['input' => 123, 'confirmation' => 0, 'conversation_token' => ''], + [ + 'output' => 456, + 'output_transcript' => 'The agent answer', + 'input_transcript' => 'Hello agent, what do you hear?', + ], + ); + $this->listener->handle(new TaskSuccessfulEvent($task)); + } + + public function testAgentReplyReceivesNoTranscriptOnConfirmationFollowUp(): void { + $sessionMapper = $this->createMock(SessionMapper::class); + $session = new Session(); + $session->setAssignmentId(null); + $session->setRequesterUserId('alice'); + $this->givenAgentSession($sessionMapper, $session); + + $this->givenUnstoredMessage(); + + /** @var AgentTalkService&MockObject $agentTalkService */ + $agentTalkService = $this->createMock(AgentTalkService::class); + $agentTalkService->expects($this->once()) + ->method('postAgentTalkReply') + ->with( + $session, + 'The agent answer', + null, + null, + 456, + [], + ); + $this->rebuildListener($agentTalkService, $sessionMapper); + + $task = $this->makeAudioTask( + ['input' => 123, 'confirmation' => 1, 'conversation_token' => 'token'], + [ + 'output' => 456, + 'output_transcript' => 'The agent answer', + 'input_transcript' => 'Hello agent, what do you hear?', + ], + ); + $this->listener->handle(new TaskSuccessfulEvent($task)); + } + + public function testAgentReplyReceivesTheMultimodalOutputFiles(): void { + $sessionMapper = $this->createMock(SessionMapper::class); + $session = new Session(); + $session->setAssignmentId(null); + $session->setRequesterUserId('alice'); + $this->givenAgentSession($sessionMapper, $session); + + $this->givenUnstoredMessage(); + + /** @var AgentTalkService&MockObject $agentTalkService */ + $agentTalkService = $this->createMock(AgentTalkService::class); + $agentTalkService->expects($this->once()) + ->method('postAgentTalkReply') + ->with( + $session, + 'Here is your image.', + null, + null, + null, + [789, 790], + ); + $this->rebuildListener($agentTalkService, $sessionMapper); + + $task = new Task( + MultimodalContextAgentInteraction::ID, + [ + 'input' => 'Generate an image of a cat', + 'confirmation' => 0, + 'conversation_token' => '', + 'input_attachments' => [], + ], + 'assistant:chatty-llm', + 'admin', + 'chatty-llm:' . self::SESSION_ID, + ); + $task->setId(self::TASK_ID); + $task->setOutput([ + 'output' => 'Here is your image.', + 'output_attachments' => [789, 790], + ]); + $this->listener->handle(new TaskSuccessfulEvent($task)); + } } diff --git a/tests/unit/Service/AgencyActionServiceTest.php b/tests/unit/Service/AgencyActionServiceTest.php new file mode 100644 index 00000000..3b00d2e5 --- /dev/null +++ b/tests/unit/Service/AgencyActionServiceTest.php @@ -0,0 +1,125 @@ +l10n = $this->createMock(IL10N::class); + $this->l10n->method('t')->willReturnCallback(static fn (string $text, array $parameters = []): string => ($parameters !== [] ? vsprintf(str_replace('%n', "\n", $text), $parameters) : $text)); + + $this->service = new AgencyActionService($this->l10n); + } + + public function testSingleActionWithKnownToolName(): void { + $pendingActions = json_encode([ + ['name' => 'send_email', 'args' => ['subject' => 'Meeting notes', 'to' => 'alice@example.org']], + ]); + + $message = $this->service->formatConfirmationMessage($pendingActions); + + $expected = "I want to perform a sensitive action:\n\n" + . "**Send an email**\n" + . "* subject: Meeting notes\n" + . '* to: alice@example.org'; + $this->assertSame($expected, $message); + } + + public function testMultipleActionsAreNumberedAndGrouped(): void { + $pendingActions = json_encode([ + ['name' => 'send_email', 'args' => ['subject' => 'Meeting notes']], + ['name' => 'delete_file', 'args' => ['path' => '/Documents/old.txt']], + ]); + + $message = $this->service->formatConfirmationMessage($pendingActions); + + $expected = "I want to perform the following sensitive actions:\n\n" + . "1. **Send an email**\n" + . "* subject: Meeting notes\n\n" + . "2. **Delete file**\n" + . '* path: /Documents/old.txt'; + $this->assertSame($expected, $message); + } + + public function testUnknownToolNameIsPrettified(): void { + $pendingActions = json_encode([ + ['name' => 'feed_the_cat', 'args' => ['cat_name' => 'Felix']], + ]); + + $message = $this->service->formatConfirmationMessage($pendingActions); + + $expected = "I want to perform a sensitive action:\n\n" + . "**Feed the cat**\n" + . '* cat name: Felix'; + $this->assertSame($expected, $message); + } + + public function testUnderscoresAreRemovedFromParameterNames(): void { + $pendingActions = json_encode([ + ['name' => 'schedule_event', 'args' => ['event_title' => 'Standup', 'start_date' => '2026-11-13']], + ]); + + $message = $this->service->formatConfirmationMessage($pendingActions); + + $this->assertStringContainsString('* event title: Standup', $message); + $this->assertStringContainsString('* start date: 2026-11-13', $message); + } + + public function testListParameterValuesAreJoined(): void { + $pendingActions = json_encode([ + ['name' => 'send_message_to_conversation', 'args' => ['recipients' => ['alice', 'bob']]], + ]); + + $message = $this->service->formatConfirmationMessage($pendingActions); + + $this->assertStringContainsString('* recipients: alice, bob', $message); + } + + public function testNewlinesInParameterValuesAreStripped(): void { + $pendingActions = json_encode([ + ['name' => 'send_email', 'args' => ['body' => "line one\nline two"]], + ]); + + $message = $this->service->formatConfirmationMessage($pendingActions); + + $this->assertStringContainsString('* body: line one line two', $message); + } + + public function testInvalidJsonReturnsNull(): void { + $this->assertNull($this->service->formatConfirmationMessage('not json')); + } + + public function testEmptyActionListReturnsNull(): void { + $this->assertNull($this->service->formatConfirmationMessage('[]')); + } + + public function testActionWithoutNameNorArgsIsSkipped(): void { + $pendingActions = json_encode([['destructive' => true]]); + + $this->assertNull($this->service->formatConfirmationMessage($pendingActions)); + } + + public function testActionDataProvidesTitlesAndIcons(): void { + $this->assertSame('Send an email', $this->service->getActionTitle('send_email')); + $this->assertSame('EmailPlus', $this->service->getActionIcon('send_email')); + $this->assertNull($this->service->getActionTitle('unknown_tool')); + $this->assertNull($this->service->getActionIcon('unknown_tool')); + } +} diff --git a/tests/unit/Service/AgentTalkFileServiceTest.php b/tests/unit/Service/AgentTalkFileServiceTest.php new file mode 100644 index 00000000..7aa06cab --- /dev/null +++ b/tests/unit/Service/AgentTalkFileServiceTest.php @@ -0,0 +1,165 @@ +service = new AgentTalkFileService( + $this->createMock(\OCP\Files\IRootFolder::class), + $this->createMock(LoggerInterface::class), + ); + $this->comment = $this->createMock(IComment::class); + } + + private function makeFileShareComment(string $message): IComment&MockObject { + $comment = $this->createMock(IComment::class); + $comment->method('getMessage')->willReturn($message); + return $comment; + } + + public function testParsePlainMessageReturnsNull(): void { + $comment = $this->makeFileShareComment('Hello agent, what is the weather?'); + $this->assertNull($this->service->parseFileShareMessage($comment)); + } + + public function testParseInvalidJsonReturnsNull(): void { + $comment = $this->makeFileShareComment('{not json'); + $this->assertNull($this->service->parseFileShareMessage($comment)); + } + + public function testParseOtherSystemMessageReturnsNull(): void { + $comment = $this->makeFileShareComment(json_encode([ + 'message' => 'thread_created', + 'parameters' => ['thread' => 42, 'title' => 'My thread'], + ])); + $this->assertNull($this->service->parseFileShareMessage($comment)); + } + + public function testParseFileSharedWithoutFileOrShareReturnsNull(): void { + $comment = $this->makeFileShareComment(json_encode([ + 'message' => 'file_shared', + 'parameters' => ['metaData' => []], + ])); + $this->assertNull($this->service->parseFileShareMessage($comment)); + } + + public function testParseFileSharedWithFileId(): void { + $comment = $this->makeFileShareComment(json_encode([ + 'message' => 'file_shared', + 'parameters' => [ + 'fileId' => '123', + 'metaData' => [ + 'messageType' => 'voice-message', + 'mimeType' => 'audio/wav', + 'caption' => 'What do you hear?', + ], + ], + ])); + $this->assertEquals([ + 'fileId' => 123, + 'shareId' => null, + 'messageType' => 'voice-message', + 'mimeType' => 'audio/wav', + 'caption' => 'What do you hear?', + ], $this->service->parseFileShareMessage($comment)); + } + + public function testParseFileSharedWithShareId(): void { + $comment = $this->makeFileShareComment(json_encode([ + 'message' => 'file_shared', + 'parameters' => [ + 'share' => 42, + 'metaData' => [ + 'mimeType' => 'image/png', + ], + ], + ])); + $this->assertEquals([ + 'fileId' => null, + 'shareId' => '42', + 'messageType' => null, + 'mimeType' => 'image/png', + 'caption' => '', + ], $this->service->parseFileShareMessage($comment)); + } + + public function testAttachmentTypeOfVoiceMessage(): void { + $fileShare = [ + 'messageType' => 'voice-message', + 'mimeType' => 'audio/wav', + ]; + $this->assertSame(AgentTalkFileService::ATTACHMENT_TYPE_AUDIO, $this->service->getAttachmentType($fileShare, null)); + } + + public function testAttachmentTypeOfAudioMimetype(): void { + $fileShare = [ + 'messageType' => null, + 'mimeType' => 'audio/mpeg', + ]; + $this->assertSame(AgentTalkFileService::ATTACHMENT_TYPE_AUDIO, $this->service->getAttachmentType($fileShare, null)); + } + + public function testAttachmentTypeOfImage(): void { + $fileShare = [ + 'messageType' => null, + 'mimeType' => 'image/png', + ]; + $this->assertSame(AgentTalkFileService::ATTACHMENT_TYPE_FILE, $this->service->getAttachmentType($fileShare, null)); + } + + public function testAttachmentTypeFromNodeMimetypeWhenMetaDataMissing(): void { + $fileShare = [ + 'messageType' => null, + 'mimeType' => null, + ]; + /** @var File&MockObject $node */ + $node = $this->createMock(File::class); + $node->method('getMimeType')->willReturn('audio/ogg'); + $this->assertSame(AgentTalkFileService::ATTACHMENT_TYPE_AUDIO, $this->service->getAttachmentType($fileShare, $node)); + } + + public function testCaptionMentionsAreParsedFromCaption(): void { + $storedMessage = ''; + $this->comment->method('setMessage')->willReturnCallback( + function (string $message) use (&$storedMessage): void { + $storedMessage = $message; + } + ); + $this->comment->method('getMentions')->willReturnCallback( + static fn (): array => [['type' => 'user', 'id' => 'agent1']] + ); + + // getCaptionMentions must set the caption as the comment message, + // like Talk's own notifier does + $fileShare = ['caption' => '@"agent1" what is on this image?']; + $mentions = $this->service->getCaptionMentions($this->comment, $fileShare); + + $this->assertSame('@"agent1" what is on this image?', $storedMessage); + $this->assertSame([['type' => 'user', 'id' => 'agent1']], $mentions); + } + + public function testCaptionMentionsWithoutCaptionAreEmpty(): void { + $fileShare = ['caption' => '']; + $this->assertSame([], $this->service->getCaptionMentions($this->comment, $fileShare)); + } +} diff --git a/tests/unit/Service/AgentUserServiceTest.php b/tests/unit/Service/AgentUserServiceTest.php new file mode 100644 index 00000000..1aefaa74 --- /dev/null +++ b/tests/unit/Service/AgentUserServiceTest.php @@ -0,0 +1,58 @@ +userManager = $this->createMock(IUserManager::class); + $this->service = new AgentUserService($this->userManager); + } + + public function testCreatesUserWithRandomPasswordAndDisplayName(): void { + $this->userManager->method('userExists')->willReturn(false); + $this->userManager->expects($this->once()) + ->method('createUser') + ->with('my_agent', $this->callback(static function (string $password) { + return strlen($password) >= 32 && ctype_alnum($password); + })) + ->willReturnCallback(function (string $userId, string $password) { + $user = $this->createConfiguredMock(IUser::class, ['getUID' => $userId]); + return $user; + }); + + $user = $this->service->createAgentUser('My Agent', 'my_agent'); + $this->assertSame('my_agent', $user->getUID()); + } + + public function testRejectsInvalidUserIds(): void { + $this->userManager->expects($this->never())->method('createUser'); + $this->expectException(AgentUserException::class); + $this->service->createAgentUser('Agent', 'invalid uid!'); + } + + public function testRejectsExistingUserIds(): void { + $this->userManager->method('userExists')->willReturn(true); + $this->userManager->expects($this->never())->method('createUser'); + $this->expectException(AgentUserException::class); + $this->service->createAgentUser('Agent', 'admin'); + } +} diff --git a/tests/unit/Service/AgentsServiceTest.php b/tests/unit/Service/AgentsServiceTest.php new file mode 100644 index 00000000..2d4a28c8 --- /dev/null +++ b/tests/unit/Service/AgentsServiceTest.php @@ -0,0 +1,226 @@ +agentMapper = $this->createMock(AgentMapper::class); + $this->agentUserService = $this->createMock(AgentUserService::class); + $this->userManager = $this->createMock(IUserManager::class); + $this->groupManager = $this->createMock(IGroupManager::class); + $this->logger = $this->createMock(LoggerInterface::class); + + $this->service = new AgentsService( + $this->agentMapper, + $this->agentUserService, + $this->userManager, + $this->groupManager, + $this->logger, + ); + } + + public function testCreateAgentProvisionsTheUserAndStoresTheAgent(): void { + $this->agentUserService->expects($this->once()) + ->method('createAgentUser') + ->with('Code Agent', 'code_agent'); + + $stored = null; + $this->agentMapper->method('insert') + ->willReturnCallback(static function ($agent) use (&$stored) { + $agent->setId(7); + $stored = $agent; + return $agent; + }); + + $agent = $this->service->createAgent('Code Agent', 'code_agent', 'Be nice.', 'tool:code'); + $this->assertSame('code_agent', $agent->getUserId()); + $this->assertSame('Be nice.', $agent->getPersonality()); + $this->assertSame('tool:code', $agent->getScope()); + $this->assertSame(7, $agent->getId()); + } + + public function testCreateAgentWithAccessList(): void { + $this->agentUserService->expects($this->once()) + ->method('createAgentUser'); + + $this->agentMapper->method('insert') + ->willReturnCallback(static function ($agent) { + $agent->setId(7); + return $agent; + }); + $this->agentMapper->expects($this->once()) + ->method('setAcl') + ->with(7, [ + ['type' => 'user', 'target' => 'user1'], + ['type' => 'group', 'target' => 'group1'], + ]); + + $this->service->createAgent('Code Agent', 'code_agent', '', '', ['user1'], ['group1']); + } + + public function testCreateAgentWithoutAccessListStoresNoAcl(): void { + $this->agentUserService->expects($this->once()) + ->method('createAgentUser'); + $this->agentMapper->method('insert') + ->willReturnCallback(static function ($agent) { + $agent->setId(7); + return $agent; + }); + $this->agentMapper->expects($this->never()) + ->method('setAcl'); + + $this->service->createAgent('Code Agent', 'code_agent', '', ''); + } + + public function testCreateAgentFailsIfUserProvisioningFails(): void { + $this->agentUserService->expects($this->once()) + ->method('createAgentUser') + ->willThrowException(new AgentUserException('taken')); + $this->agentMapper->expects($this->never())->method('insert'); + + $this->expectException(AgentUserException::class); + $this->service->createAgent('Code Agent', 'code_agent', 'Be nice.', ''); + } + + public function testDeleteAgentWithoutUserDeletion(): void { + $agent = $this->createAgentEntity(5, 'agent1'); + $this->agentMapper->method('getById')->willReturn($agent); + $this->agentMapper->expects($this->once())->method('deleteAgent')->with($agent); + $this->userManager->expects($this->never())->method('get'); + + $this->service->deleteAgent(5, false); + } + + public function testDeleteAgentWithUserDeletion(): void { + $agent = $this->createAgentEntity(5, 'agent1'); + $this->agentMapper->method('getById')->willReturn($agent); + $this->agentMapper->expects($this->once())->method('deleteAgent')->with($agent); + + $user = $this->createMock(IUser::class); + $user->expects($this->once())->method('delete'); + $this->userManager->expects($this->once())->method('get')->with('agent1')->willReturn($user); + + $this->service->deleteAgent(5, true); + } + + public function testDeleteAgentKeepsTheUserIfDeleteFails(): void { + $agent = $this->createAgentEntity(5, 'agent1'); + $this->agentMapper->method('getById')->willReturn($agent); + $this->agentMapper->expects($this->once())->method('deleteAgent') + ->willThrowException(new \OCP\DB\Exception('db down')); + $this->userManager->expects($this->never())->method('get'); + + $this->expectException(\OCP\DB\Exception::class); + $this->service->deleteAgent(5, true); + } + + public function testUpdateAgent() { + $agent = $this->createAgentEntity(5, 'agent1'); + $this->agentMapper->method('getById')->willReturn($agent); + $this->agentMapper->expects($this->once())->method('update') + ->willReturnCallback(static function ($updated) { + return $updated; + }); + + $updated = $this->service->updateAgent(5, personality: 'New personality'); + $this->assertSame('New personality', $updated->getPersonality()); + } + + public function testUpdateAgentKeepsTheAccessWhenNotProvided(): void { + $agent = $this->createAgentEntity(5, 'agent1'); + $this->agentMapper->method('getById')->willReturn($agent); + $this->agentMapper->method('update') + ->willReturnCallback(static function ($updated) { + return $updated; + }); + $this->agentMapper->expects($this->never())->method('setAcl'); + $this->agentMapper->expects($this->never())->method('getAcl'); + + $this->service->updateAgent(5, personality: 'New personality'); + } + + public function testUpdateAgentReplacesTheUsersAccessOnly(): void { + $agent = $this->createAgentEntity(5, 'agent1'); + $this->agentMapper->method('getById')->willReturn($agent); + $this->agentMapper->method('update') + ->willReturnCallback(static function ($updated) { + return $updated; + }); + $this->agentMapper->expects($this->once())->method('getAcl') + ->with(5) + ->willReturn([ + ['type' => 'user', 'target' => 'user1'], + ['type' => 'group', 'target' => 'group1'], + ]); + $this->agentMapper->expects($this->once())->method('setAcl') + ->with(5, [ + ['type' => 'user', 'target' => 'user2'], + ['type' => 'group', 'target' => 'group1'], + ]); + + $this->service->updateAgent(5, accessUsers: ['user2']); + } + + public function testCanUserAccessAgentChecksTheUserGroups(): void { + $agent = $this->createAgentEntity(5, 'agent1'); + $user = $this->createMock(IUser::class); + $user->method('getUID')->willReturn('user1'); + $this->groupManager->method('getUserGroupIds')->with($user)->willReturn(['group1']); + $this->agentMapper->expects($this->once()) + ->method('canUserAccess') + ->with(5, 'user1', ['group1']) + ->willReturn(true); + + $this->assertTrue($this->service->canUserAccessAgent($agent, $user)); + } + + public function testGetAgentAccessSplitsUsersAndGroups(): void { + $this->agentMapper->expects($this->once()) + ->method('getAcl') + ->with(5) + ->willReturn([ + ['type' => 'user', 'target' => 'user1'], + ['type' => 'group', 'target' => 'group1'], + ['type' => 'user', 'target' => 'user2'], + ]); + + $access = $this->service->getAgentAccess(5); + $this->assertSame(['user1', 'user2'], $access['users']); + $this->assertSame(['group1'], $access['groups']); + } + + private function createAgentEntity(int $id, string $userId): \OCA\Assistant\Db\Agent { + $agent = new \OCA\Assistant\Db\Agent(); + $agent->setId($id); + $agent->setUserId($userId); + $agent->setPersonality('Be nice.'); + return $agent; + } +} diff --git a/tests/unit/Service/ChatServiceAgencyTest.php b/tests/unit/Service/ChatServiceAgencyTest.php index 97aaac02..31d59e8c 100644 --- a/tests/unit/Service/ChatServiceAgencyTest.php +++ b/tests/unit/Service/ChatServiceAgencyTest.php @@ -42,16 +42,24 @@ class ChatServiceAgencyTest extends TestCase { private ChatService $service; private MessageMapper&MockObject $messageMapper; private IManager&MockObject $taskProcessingManager; + private SessionMapper&MockObject $sessionMapper; + private SessionSummaryService&MockObject $sessionSummaryService; + private IUserManager&MockObject $userManager; + private IAppConfig&MockObject $appConfig; + private IL10N&MockObject $l10n; + private IDBConnection&MockObject $db; + private LoggerInterface&MockObject $logger; + private ITimeFactory&MockObject $timeFactory; private ?Task $scheduledTask = null; protected function setUp(): void { parent::setUp(); - $sessionMapper = $this->createMock(SessionMapper::class); - $sessionMapper->method('exists')->willReturn(true); + $this->sessionMapper = $this->createMock(SessionMapper::class); + $this->sessionMapper->method('exists')->willReturn(true); $session = new Session(); $session->setUserId(self::USER_ID); - $sessionMapper->method('getUserSession')->willReturn($session); + $this->sessionMapper->method('getUserSession')->willReturn($session); $humanMessage = new Message(); $humanMessage->setSessionId(self::SESSION_ID); @@ -69,21 +77,49 @@ protected function setUp(): void { $this->scheduledTask = $task; }); + $this->sessionSummaryService = $this->createMock(SessionSummaryService::class); + + $l10n = $this->createMock(IL10N::class); + $l10n->method('t')->willReturnCallback(static fn (string $text, array $parameters = []): string => ($parameters !== [] ? vsprintf(str_replace('%n', "\n", $text), $parameters) : $text)); + + $this->l10n = $l10n; + $this->userManager = $this->createMock(IUserManager::class); + $this->appConfig = $this->createMock(IAppConfig::class); + $this->db = $this->createMock(IDBConnection::class); + $this->logger = $this->createMock(LoggerInterface::class); + $this->timeFactory = $this->createMock(ITimeFactory::class); + + $this->rebuildService(); + } + + /** + * The service must be rebuilt when the mocked SessionMapper changes: + * re-stubbing an already configured mock method would not override + * the previously configured return value. + */ + private function rebuildService(): void { $this->service = new ChatService( - $this->createMock(IUserManager::class), - $this->createMock(IAppConfig::class), + $this->userManager, + $this->appConfig, $this->createMock(IConfig::class), - $this->createMock(IL10N::class), - $sessionMapper, + $this->l10n, + $this->sessionMapper, $this->messageMapper, - $this->createMock(IDBConnection::class), - $this->createMock(SessionSummaryService::class), + $this->db, + $this->sessionSummaryService, $this->taskProcessingManager, - $this->createMock(LoggerInterface::class), - $this->createMock(ITimeFactory::class), + $this->logger, + $this->timeFactory, ); } + private function setUserSession(Session $session): void { + $this->sessionMapper = $this->createMock(SessionMapper::class); + $this->sessionMapper->method('exists')->willReturn(true); + $this->sessionMapper->method('getUserSession')->willReturn($session); + $this->rebuildService(); + } + private function setAgentOptionalInputs(array $optionalInputShape): void { $this->taskProcessingManager->method('getAvailableTaskTypes')->willReturn([ ContextAgentInteraction::ID => ['optionalInputShape' => $optionalInputShape], @@ -153,4 +189,72 @@ public function testUserInstructionsAreNotSentForAssignmentRuns(): void { $this->assertSame(ContextAgentInteraction::ID, $this->scheduledTask->getTaskTypeId()); $this->assertArrayNotHasKey('user_instructions', $this->scheduledTask->getInput()); } + + public function testTaskIsScheduledAsTheSessionOwner(): void { + $this->setAgentOptionalInputs([]); + + $this->service->scheduleMessageGeneration(self::USER_ID, self::SESSION_ID); + + $this->assertNotNull($this->scheduledTask); + $this->assertSame(self::USER_ID, $this->scheduledTask->getUserId()); + } + + public function testAgentSessionsScheduleAsTheAgentOwnerWithTheRequesterMemories(): void { + $agentUserId = 'agent_user_id'; + $session = new Session(); + $session->setUserId($agentUserId); + $session->setRequesterUserId(self::USER_ID); + $this->setUserSession($session); + $this->setAgentOptionalInputs(['memories' => 'shape']); + $this->messageMapper->method('getFirstNMessages')->willThrowException(new DoesNotExistException('no system message')); + $this->sessionSummaryService->expects($this->once()) + ->method('getMemories') + ->with(self::USER_ID) + ->willReturn([]); + + $this->service->scheduleMessageGeneration($agentUserId, self::SESSION_ID); + + $this->assertNotNull($this->scheduledTask); + $this->assertSame($agentUserId, $this->scheduledTask->getUserId()); + $this->assertArrayHasKey('memories', $this->scheduledTask->getInput()); + } + + /** + * The generation tasks of agent sessions must describe the human requester + * and the agent: without it, the model would not know either identity. + */ + public function testAgentSessionsGetTheIdentityLine(): void { + $session = new Session(); + $session->setUserId('agent_user_id'); + $session->setRequesterUserId(self::USER_ID); + $this->setUserSession($session); + $this->setAgentOptionalInputs(['user_instructions' => 'shape']); + $this->messageMapper->method('getFirstNMessages') + ->with(self::SESSION_ID, 1, Message::ROLE_SYSTEM) + ->willReturn($this->systemMessage(self::USER_INSTRUCTIONS)); + + $this->service->scheduleMessageGeneration('agent_user_id', self::SESSION_ID, 0, 'My Group Room'); + + $this->assertNotNull($this->scheduledTask); + $userInstructions = $this->scheduledTask->getInput()['user_instructions'] ?? null; + $this->assertNotNull($userInstructions); + // the agent identity and the requester identity are both in the line + $this->assertStringContainsString('agent_user_id', $userInstructions); + $this->assertStringContainsString(self::USER_ID, $userInstructions); + $this->assertStringContainsString('My Group Room', $userInstructions); + // the session instructions must still be part of the agent context + $this->assertStringContainsString(self::USER_INSTRUCTIONS, $userInstructions); + } + + public function testHumanSessionsGetNoIdentityLine(): void { + $this->setAgentOptionalInputs(['user_instructions' => 'shape']); + $this->messageMapper->method('getFirstNMessages') + ->with(self::SESSION_ID, 1, Message::ROLE_SYSTEM) + ->willReturn($this->systemMessage(self::USER_INSTRUCTIONS)); + + $this->service->scheduleMessageGeneration(self::USER_ID, self::SESSION_ID); + + $this->assertNotNull($this->scheduledTask); + $this->assertSame(self::USER_INSTRUCTIONS, $this->scheduledTask->getInput()['user_instructions'] ?? null); + } }