Joe Danziger e3ed4d7c14
feat: CLI & MCP progress tracking for parse-prd command (#1048)
* initial cutover

* update log to debug

* update tracker to pass units

* update test to match new base tracker format

* add streamTextService mocks

* remove unused imports

* Ensure the CLI waits for async main() completion

* refactor to reduce code duplication

* update comment

* reuse function

* ensure targetTag is defined in streaming mode

* avoid throwing inside process.exit spy

* check for null

* remove reference to generate

* fix formatting

* fix textStream assignment

* ensure no division by 0

* fix jest chalk mocks

* refactor for maintainability

* Improve bar chart calculation logic for consistent visual representation

* use custom streaming error types; fix mocks

* Update streamText extraction in parse-prd.js to match actual service response

* remove check - doesn't belong here

* update mocks

* remove streaming test that wasn't really doing anything

* add comment

* make parsing logic more DRY

* fix formatting

* Fix textStream extraction to match actual service response

* fix mock

* Add a cleanup method to ensure proper resource disposal and prevent memory leaks

* debounce progress updates to reduce UI flicker during rapid updates

* Implement timeout protection for streaming operations (60-second timeout) with automatic fallback to non-streaming mode.

* clear timeout properly

* Add a maximum buffer size limit (1MB) to prevent unbounded memory growth with very large streaming responses.

* fix formatting

* remove duplicate mock

* better docs

* fix formatting

* sanitize the dynamic property name

* Fix incorrect remaining progress calculation

* Use onError callback instead of console.warn

* Remove unused chalk import

* Add missing custom validator in fallback parsing configuration

* add custom validator parameter in fallback parsing

* chore: fix package-lock.json

* chore: large code refactor

* chore: increase timeout from 1 minute to 3 minutes

* fix: refactor and fix streaming

* Merge remote-tracking branch 'origin/next' into joedanz/parse-prd-progress

* fix: cleanup and fix unit tests

* chore: fix unit tests

* chore: fix format

* chore: run format

* chore: fix weird CI unit test error

* chore: fix format

---------

Co-authored-by: Ralph Khreish <35776126+Crunchyman-ralph@users.noreply.github.com>
2025-08-12 22:37:07 +02:00

385 lines
10 KiB
JavaScript

/**
* Helper functions for PRD parsing
*/
import fs from 'fs';
import path from 'path';
import boxen from 'boxen';
import chalk from 'chalk';
import { ensureTagMetadata, findTaskById } from '../../utils.js';
import { getPriorityIndicators } from '../../../../src/ui/indicators.js';
import { displayParsePrdSummary } from '../../../../src/ui/parse-prd.js';
import { TimeoutManager } from '../../../../src/utils/timeout-manager.js';
import { displayAiUsageSummary } from '../../ui.js';
import { getPromptManager } from '../../prompt-manager.js';
import { getDefaultPriority } from '../../config-manager.js';
/**
* Estimate token count from text
* @param {string} text - Text to estimate tokens for
* @returns {number} Estimated token count
*/
export function estimateTokens(text) {
// Common approximation: ~4 characters per token for English
return Math.ceil(text.length / 4);
}
/**
* Read and validate PRD content
* @param {string} prdPath - Path to PRD file
* @returns {string} PRD content
* @throws {Error} If file is empty or cannot be read
*/
export function readPrdContent(prdPath) {
const prdContent = fs.readFileSync(prdPath, 'utf8');
if (!prdContent) {
throw new Error(`Input file ${prdPath} is empty or could not be read.`);
}
return prdContent;
}
/**
* Load existing tasks from file
* @param {string} tasksPath - Path to tasks file
* @param {string} targetTag - Target tag to load from
* @returns {{tasks: Array, nextId: number}} Existing tasks and next ID
*/
export function loadExistingTasks(tasksPath, targetTag) {
let existingTasks = [];
let nextId = 1;
if (!fs.existsSync(tasksPath)) {
return { existingTasks, nextId };
}
try {
const existingFileContent = fs.readFileSync(tasksPath, 'utf8');
const allData = JSON.parse(existingFileContent);
if (allData[targetTag]?.tasks && Array.isArray(allData[targetTag].tasks)) {
existingTasks = allData[targetTag].tasks;
if (existingTasks.length > 0) {
nextId = Math.max(...existingTasks.map((t) => t.id || 0)) + 1;
}
}
} catch (error) {
// If we can't read the file or parse it, assume no existing tasks
return { existingTasks: [], nextId: 1 };
}
return { existingTasks, nextId };
}
/**
* Validate overwrite/append operations
* @param {Object} params
* @returns {void}
* @throws {Error} If validation fails
*/
export function validateFileOperations({
existingTasks,
targetTag,
append,
force,
isMCP,
logger
}) {
const hasExistingTasks = existingTasks.length > 0;
if (!hasExistingTasks) {
logger.report(
`Tag '${targetTag}' is empty or doesn't exist. Creating/updating tag with new tasks.`,
'info'
);
return;
}
if (append) {
logger.report(
`Append mode enabled. Found ${existingTasks.length} existing tasks in tag '${targetTag}'.`,
'info'
);
return;
}
if (!force) {
const errorMessage = `Tag '${targetTag}' already contains ${existingTasks.length} tasks. Use --force to overwrite or --append to add to existing tasks.`;
logger.report(errorMessage, 'error');
if (isMCP) {
throw new Error(errorMessage);
} else {
console.error(chalk.red(errorMessage));
process.exit(1);
}
}
logger.report(
`Force flag enabled. Overwriting existing tasks in tag '${targetTag}'.`,
'debug'
);
}
/**
* Process and transform tasks with ID remapping
* @param {Array} rawTasks - Raw tasks from AI
* @param {number} startId - Starting ID for new tasks
* @param {Array} existingTasks - Existing tasks for dependency validation
* @param {string} defaultPriority - Default priority for tasks
* @returns {Array} Processed tasks with remapped IDs
*/
export function processTasks(
rawTasks,
startId,
existingTasks,
defaultPriority
) {
let currentId = startId;
const taskMap = new Map();
// First pass: assign new IDs and create mapping
const processedTasks = rawTasks.map((task) => {
const newId = currentId++;
taskMap.set(task.id, newId);
return {
...task,
id: newId,
status: task.status || 'pending',
priority: task.priority || defaultPriority,
dependencies: Array.isArray(task.dependencies) ? task.dependencies : [],
subtasks: task.subtasks || [],
// Ensure all required fields have values
title: task.title || '',
description: task.description || '',
details: task.details || '',
testStrategy: task.testStrategy || ''
};
});
// Second pass: remap dependencies
processedTasks.forEach((task) => {
task.dependencies = task.dependencies
.map((depId) => taskMap.get(depId))
.filter(
(newDepId) =>
newDepId != null &&
newDepId < task.id &&
(findTaskById(existingTasks, newDepId) ||
processedTasks.some((t) => t.id === newDepId))
);
});
return processedTasks;
}
/**
* Save tasks to file with tag support
* @param {string} tasksPath - Path to save tasks
* @param {Array} tasks - Tasks to save
* @param {string} targetTag - Target tag
* @param {Object} logger - Logger instance
*/
export function saveTasksToFile(tasksPath, tasks, targetTag, logger) {
// Create directory if it doesn't exist
const tasksDir = path.dirname(tasksPath);
if (!fs.existsSync(tasksDir)) {
fs.mkdirSync(tasksDir, { recursive: true });
}
// Read existing file to preserve other tags
let outputData = {};
if (fs.existsSync(tasksPath)) {
try {
const existingFileContent = fs.readFileSync(tasksPath, 'utf8');
outputData = JSON.parse(existingFileContent);
} catch (error) {
outputData = {};
}
}
// Update only the target tag
outputData[targetTag] = {
tasks: tasks,
metadata: {
created:
outputData[targetTag]?.metadata?.created || new Date().toISOString(),
updated: new Date().toISOString(),
description: `Tasks for ${targetTag} context`
}
};
// Ensure proper metadata
ensureTagMetadata(outputData[targetTag], {
description: `Tasks for ${targetTag} context`
});
// Write back to file
fs.writeFileSync(tasksPath, JSON.stringify(outputData, null, 2));
logger.report(
`Successfully saved ${tasks.length} tasks to ${tasksPath}`,
'debug'
);
}
/**
* Build prompts for AI service
* @param {Object} config - Configuration object
* @param {string} prdContent - PRD content
* @param {number} nextId - Next task ID
* @returns {Promise<{systemPrompt: string, userPrompt: string}>}
*/
export async function buildPrompts(config, prdContent, nextId) {
const promptManager = getPromptManager();
const defaultTaskPriority =
getDefaultPriority(config.projectRoot) || 'medium';
return promptManager.loadPrompt('parse-prd', {
research: config.research,
numTasks: config.numTasks,
nextId,
prdContent,
prdPath: config.prdPath,
defaultTaskPriority,
isClaudeCode: config.isClaudeCode(),
projectRoot: config.projectRoot || ''
});
}
/**
* Handle progress reporting for both CLI and MCP
* @param {Object} params
*/
export async function reportTaskProgress({
task,
currentCount,
totalTasks,
estimatedTokens,
progressTracker,
reportProgress,
priorityMap,
defaultPriority,
estimatedInputTokens
}) {
const priority = task.priority || defaultPriority;
const priorityIndicator = priorityMap[priority] || priorityMap.medium;
// CLI progress tracker
if (progressTracker) {
progressTracker.addTaskLine(currentCount, task.title, priority);
if (estimatedTokens) {
progressTracker.updateTokens(estimatedInputTokens, estimatedTokens);
}
}
// MCP progress reporting
if (reportProgress) {
try {
const outputTokens = estimatedTokens
? Math.floor(estimatedTokens / totalTasks)
: 0;
await reportProgress({
progress: currentCount,
total: totalTasks,
message: `${priorityIndicator} Task ${currentCount}/${totalTasks} - ${task.title} | ~Output: ${outputTokens} tokens`
});
} catch (error) {
// Ignore progress reporting errors
}
}
}
/**
* Display completion summary for CLI
* @param {Object} params
*/
export async function displayCliSummary({
processedTasks,
nextId,
summary,
prdPath,
tasksPath,
usedFallback,
aiServiceResponse
}) {
// Generate task file names
const taskFilesGenerated = (() => {
if (!Array.isArray(processedTasks) || processedTasks.length === 0) {
return `task_${String(nextId).padStart(3, '0')}.txt`;
}
const firstNewTaskId = processedTasks[0].id;
const lastNewTaskId = processedTasks[processedTasks.length - 1].id;
if (processedTasks.length === 1) {
return `task_${String(firstNewTaskId).padStart(3, '0')}.txt`;
}
return `task_${String(firstNewTaskId).padStart(3, '0')}.txt -> task_${String(lastNewTaskId).padStart(3, '0')}.txt`;
})();
displayParsePrdSummary({
totalTasks: processedTasks.length,
taskPriorities: summary.taskPriorities,
prdFilePath: prdPath,
outputPath: tasksPath,
elapsedTime: summary.elapsedTime,
usedFallback,
taskFilesGenerated,
actionVerb: summary.actionVerb
});
// Display telemetry
if (aiServiceResponse?.telemetryData) {
// For streaming, wait briefly to allow usage data to be captured
if (aiServiceResponse.mainResult?.usage) {
// Give the usage promise a short time to resolve
await TimeoutManager.withSoftTimeout(
aiServiceResponse.mainResult.usage,
1000,
undefined
);
}
displayAiUsageSummary(aiServiceResponse.telemetryData, 'cli');
}
}
/**
* Display non-streaming CLI output
* @param {Object} params
*/
export function displayNonStreamingCliOutput({
processedTasks,
research,
finalTasks,
tasksPath,
aiServiceResponse
}) {
console.log(
boxen(
chalk.green(
`Successfully generated ${processedTasks.length} new tasks${research ? ' with research-backed analysis' : ''}. Total tasks in ${tasksPath}: ${finalTasks.length}`
),
{ padding: 1, borderColor: 'green', borderStyle: 'round' }
)
);
console.log(
boxen(
chalk.white.bold('Next Steps:') +
'\n\n' +
`${chalk.cyan('1.')} Run ${chalk.yellow('task-master list')} to view all tasks\n` +
`${chalk.cyan('2.')} Run ${chalk.yellow('task-master expand --id=<id>')} to break down a task into subtasks`,
{
padding: 1,
borderColor: 'cyan',
borderStyle: 'round',
margin: { top: 1 }
}
)
);
if (aiServiceResponse?.telemetryData) {
displayAiUsageSummary(aiServiceResponse.telemetryData, 'cli');
}
}