botSourceMapper = $botSourceMapper; $this->embeddingMapper = $embeddingMapper; $this->rootFolder = $rootFolder; $this->logger = $logger; // Run every 6 hours $this->setInterval(6 * 60 * 60); } protected function run($arguments): void { $this->logger->info('EducAI: Starting orphaned RAG sources cleanup'); $cleanedCount = 0; $errorCount = 0; try { $sources = $this->botSourceMapper->findAll(); foreach ($sources as $source) { try { $this->checkAndCleanSource($source); } catch (Exception $e) { $errorCount++; $this->logger->warning('EducAI: Error checking source for cleanup', [ 'sourceId' => $source->getId(), 'error' => $e->getMessage(), ]); } } $this->logger->info('EducAI: Orphaned sources cleanup completed', [ 'cleaned' => $cleanedCount, 'errors' => $errorCount, ]); } catch (Exception $e) { $this->logger->error('EducAI: Failed to run orphaned sources cleanup', [ 'exception' => $e, ]); } } private function checkAndCleanSource(BotSource $source): bool { // Skip sources already marked as error with "no longer exists" message if ($source->getStatus() === 'error' && str_contains($source->getErrorMessage() ?? '', 'no longer exists')) { return false; } // Skip URL sources - they don't have a file to check // URL sources have node_type = 'url' and node_id = 0 if ($source->getNodeType() === 'url') { return false; } $ownerUid = $source->getOwnerUid(); if ($ownerUid === null || $ownerUid === '') { return false; } try { $userFolder = $this->rootFolder->getUserFolder($ownerUid); $nodes = $userFolder->getById($source->getNodeId()); // If nodes array is empty or node doesn't exist, the file was deleted if (!is_array($nodes) || count($nodes) === 0) { $this->cleanupDeletedSource($source); return true; } } catch (Exception $e) { // User might not exist anymore, or other filesystem issues // Mark as error but don't delete the source record $this->logger->info('EducAI: Cannot access source, marking as orphaned', [ 'sourceId' => $source->getId(), 'nodeId' => $source->getNodeId(), 'ownerUid' => $ownerUid, 'error' => $e->getMessage(), ]); $this->cleanupDeletedSource($source); return true; } return false; } private function cleanupDeletedSource(BotSource $source): void { $embeddingCount = count($this->embeddingMapper->findBySource($source->getId())); // Delete all embeddings for this source $this->embeddingMapper->deleteBySource($source->getId()); // Update source status $source->setStatus('error'); $source->setErrorMessage('File or folder no longer exists. ' . $embeddingCount . ' embedding(s) cleaned up. You can remove this source.'); $source->setChecksum(null); $source->setUpdatedAt(time()); $source->setProgress(null); $source->setProgressCurrent(null); $source->setProgressTotal(null); $source->setProgressStage(null); $this->botSourceMapper->update($source); $this->logger->info('EducAI: Cleaned up orphaned source', [ 'sourceId' => $source->getId(), 'nodeId' => $source->getNodeId(), 'embeddingsDeleted' => $embeddingCount, ]); } }