Markdown Dynamic Content Generation: Complete Guide for Automated Documentation and Data-Driven Content Creation
Dynamic Markdown content generation transforms static documentation workflows into powerful, data-driven systems that automatically create, update, and maintain content based on external data sources, templates, and programmatic logic. By implementing sophisticated templating systems, data binding mechanisms, and automated generation pipelines, development teams can create scalable documentation architectures that ensure content accuracy, reduce manual maintenance overhead, and enable real-time synchronization between code changes and documentation updates.
Why Master Dynamic Content Generation?
Dynamic content generation provides essential capabilities for modern documentation systems:
- Automated Synchronization: Keep documentation automatically synchronized with code changes, API updates, and system configurations
- Scalability: Generate thousands of documentation pages from structured data without manual writing
- Consistency: Ensure uniform formatting, structure, and content patterns across all generated documentation
- Real-time Updates: Automatically update content when underlying data sources change
- Reduced Maintenance: Eliminate manual documentation maintenance through programmatic content generation
Foundation Dynamic Content Concepts
Core Template Engine Implementation
Building a flexible template engine for Markdown content generation:
// markdown-template-engine.js - Dynamic content generation system
class MarkdownTemplateEngine {
constructor(options = {}) {
this.options = {
templateDirectory: options.templateDirectory || './templates',
outputDirectory: options.outputDirectory || './output',
dataDirectory: options.dataDirectory || './data',
partialDirectory: options.partialDirectory || './partials',
cacheTemplates: options.cacheTemplates !== false,
enableIncludes: options.enableIncludes !== false,
enableHelpers: options.enableHelpers !== false,
dateFormat: options.dateFormat || 'YYYY-MM-DD',
encoding: options.encoding || 'utf-8',
...options
};
this.templateCache = new Map();
this.partialCache = new Map();
this.dataCache = new Map();
this.helpers = new Map();
this.globals = new Map();
this.setupDefaultHelpers();
this.setupDefaultFilters();
}
setupDefaultHelpers() {
// Date formatting helper
this.registerHelper('formatDate', (date, format = this.options.dateFormat) => {
if (!date) return '';
const dateObj = new Date(date);
return this.formatDate(dateObj, format);
});
// Conditional helper
this.registerHelper('if', (condition, trueValue, falseValue = '') => {
return condition ? trueValue : falseValue;
});
// Loop helper
this.registerHelper('each', (items, template) => {
if (!Array.isArray(items)) return '';
return items.map((item, index) => {
const context = {
...item,
'@index': index,
'@first': index === 0,
'@last': index === items.length - 1,
'@length': items.length
};
return this.processTemplate(template, context);
}).join('\n');
});
// Join helper
this.registerHelper('join', (items, separator = ', ') => {
if (!Array.isArray(items)) return '';
return items.join(separator);
});
// Uppercase helper
this.registerHelper('upper', (text) => {
return typeof text === 'string' ? text.toUpperCase() : text;
});
// Lowercase helper
this.registerHelper('lower', (text) => {
return typeof text === 'string' ? text.toLowerCase() : text;
});
// Title case helper
this.registerHelper('title', (text) => {
return typeof text === 'string'
? text.replace(/\w\S*/g, (txt) => txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase())
: text;
});
// Markdown escape helper
this.registerHelper('escape', (text) => {
if (typeof text !== 'string') return text;
return text.replace(/([\\`*_{}\[\]()#+\-.!])/g, '\\$1');
});
// Code block helper
this.registerHelper('codeblock', (code, language = '') => {
return `\`\`\`${language}\n${code}\n\`\`\``;
});
// Link helper
this.registerHelper('link', (text, url, title = '') => {
const titleAttr = title ? ` "${title}"` : '';
return `[${text}](${url}${titleAttr})`;
});
// Image helper
this.registerHelper('image', (alt, src, title = '') => {
const titleAttr = title ? ` "${title}"` : '';
return ``;
});
// Table helper
this.registerHelper('table', (data, headers = null) => {
if (!Array.isArray(data) || data.length === 0) return '';
const actualHeaders = headers || Object.keys(data[0]);
let table = `| ${actualHeaders.join(' | ')} |\n`;
table += `|${actualHeaders.map(() => '---').join('|')}|\n`;
data.forEach(row => {
const values = actualHeaders.map(header => row[header] || '');
table += `| ${values.join(' | ')} |\n`;
});
return table;
});
}
setupDefaultFilters() {
this.filters = new Map();
this.filters.set('length', (value) => {
if (Array.isArray(value) || typeof value === 'string') {
return value.length;
}
return 0;
});
this.filters.set('first', (array, count = 1) => {
if (!Array.isArray(array)) return [];
return count === 1 ? array[0] : array.slice(0, count);
});
this.filters.set('last', (array, count = 1) => {
if (!Array.isArray(array)) return [];
return count === 1 ? array[array.length - 1] : array.slice(-count);
});
this.filters.set('sort', (array, key = null) => {
if (!Array.isArray(array)) return array;
return array.slice().sort((a, b) => {
const valueA = key ? a[key] : a;
const valueB = key ? b[key] : b;
if (typeof valueA === 'string' && typeof valueB === 'string') {
return valueA.localeCompare(valueB);
}
return valueA < valueB ? -1 : valueA > valueB ? 1 : 0;
});
});
this.filters.set('reverse', (array) => {
if (!Array.isArray(array)) return array;
return array.slice().reverse();
});
this.filters.set('unique', (array, key = null) => {
if (!Array.isArray(array)) return array;
const seen = new Set();
return array.filter(item => {
const value = key ? item[key] : item;
if (seen.has(value)) return false;
seen.add(value);
return true;
});
});
this.filters.set('where', (array, key, value) => {
if (!Array.isArray(array)) return array;
return array.filter(item => item[key] === value);
});
}
registerHelper(name, helperFunction) {
this.helpers.set(name, helperFunction);
}
registerGlobal(name, value) {
this.globals.set(name, value);
}
async loadTemplate(templateName) {
if (this.options.cacheTemplates && this.templateCache.has(templateName)) {
return this.templateCache.get(templateName);
}
const fs = require('fs').promises;
const path = require('path');
const templatePath = path.join(this.options.templateDirectory, `${templateName}.md`);
try {
const templateContent = await fs.readFile(templatePath, this.options.encoding);
if (this.options.cacheTemplates) {
this.templateCache.set(templateName, templateContent);
}
return templateContent;
} catch (error) {
throw new Error(`Template '${templateName}' not found at ${templatePath}: ${error.message}`);
}
}
async loadPartial(partialName) {
if (this.options.cacheTemplates && this.partialCache.has(partialName)) {
return this.partialCache.get(partialName);
}
const fs = require('fs').promises;
const path = require('path');
const partialPath = path.join(this.options.partialDirectory, `${partialName}.md`);
try {
const partialContent = await fs.readFile(partialPath, this.options.encoding);
if (this.options.cacheTemplates) {
this.partialCache.set(partialName, partialContent);
}
return partialContent;
} catch (error) {
throw new Error(`Partial '${partialName}' not found at ${partialPath}: ${error.message}`);
}
}
async loadData(dataName) {
if (this.options.cacheTemplates && this.dataCache.has(dataName)) {
return this.dataCache.get(dataName);
}
const fs = require('fs').promises;
const path = require('path');
// Try different file extensions
const extensions = ['.json', '.yaml', '.yml', '.js'];
for (const ext of extensions) {
const dataPath = path.join(this.options.dataDirectory, `${dataName}${ext}`);
try {
let data;
if (ext === '.json') {
const content = await fs.readFile(dataPath, this.options.encoding);
data = JSON.parse(content);
} else if (ext === '.yaml' || ext === '.yml') {
const yaml = require('yaml');
const content = await fs.readFile(dataPath, this.options.encoding);
data = yaml.parse(content);
} else if (ext === '.js') {
delete require.cache[require.resolve(path.resolve(dataPath))];
data = require(path.resolve(dataPath));
}
if (this.options.cacheTemplates) {
this.dataCache.set(dataName, data);
}
return data;
} catch (error) {
continue; // Try next extension
}
}
throw new Error(`Data file '${dataName}' not found with extensions: ${extensions.join(', ')}`);
}
async processTemplate(template, context = {}) {
const mergedContext = {
...Object.fromEntries(this.globals),
...context
};
let processed = template;
// Process includes first
if (this.options.enableIncludes) {
processed = await this.processIncludes(processed, mergedContext);
}
// Process template variables and expressions
processed = this.processVariables(processed, mergedContext);
// Process helpers
if (this.options.enableHelpers) {
processed = this.processHelpers(processed, mergedContext);
}
// Process filters
processed = this.processFilters(processed, mergedContext);
return processed;
}
async processIncludes(template, context) {
const includePattern = /{{>\s*(\w+)(?:\s+(.+?))?\s*}}/g;
let processed = template;
let match;
while ((match = includePattern.exec(template)) !== null) {
const partialName = match[1];
const partialContextStr = match[2];
try {
const partial = await this.loadPartial(partialName);
let partialContext = context;
// Parse additional context for partial
if (partialContextStr) {
const additionalContext = this.parseContextExpression(partialContextStr, context);
partialContext = { ...context, ...additionalContext };
}
const processedPartial = await this.processTemplate(partial, partialContext);
processed = processed.replace(match[0], processedPartial);
} catch (error) {
console.warn(`Include error: ${error.message}`);
processed = processed.replace(match[0], `<!-- Include error: ${error.message} -->`);
}
}
return processed;
}
processVariables(template, context) {
// Simple variable substitution: {{variable}}
const variablePattern = /{{(\w+(?:\.\w+)*)}}/g;
return template.replace(variablePattern, (match, variablePath) => {
const value = this.resolveVariable(variablePath, context);
return value !== undefined ? String(value) : match;
});
}
processHelpers(template, context) {
// Helper pattern: {{helper_name arg1 arg2 ...}}
const helperPattern = /{{(\w+)\s+([^}]+)}}/g;
return template.replace(helperPattern, (match, helperName, argsStr) => {
if (!this.helpers.has(helperName)) {
return match;
}
try {
const helper = this.helpers.get(helperName);
const args = this.parseArguments(argsStr, context);
const result = helper(...args);
return result !== undefined ? String(result) : '';
} catch (error) {
console.warn(`Helper error in ${helperName}: ${error.message}`);
return `<!-- Helper error: ${error.message} -->`;
}
});
}
processFilters(template, context) {
// Filter pattern: {{variable | filter1 | filter2:arg}}
const filterPattern = /{{([^}]+\|[^}]+)}}/g;
return template.replace(filterPattern, (match, expression) => {
try {
const result = this.evaluateFilterExpression(expression, context);
return result !== undefined ? String(result) : '';
} catch (error) {
console.warn(`Filter error: ${error.message}`);
return `<!-- Filter error: ${error.message} -->`;
}
});
}
evaluateFilterExpression(expression, context) {
const parts = expression.split('|').map(part => part.trim());
let value = this.resolveVariable(parts[0], context);
for (let i = 1; i < parts.length; i++) {
const filterPart = parts[i];
const [filterName, ...filterArgs] = filterPart.split(':');
if (!this.filters.has(filterName.trim())) {
throw new Error(`Unknown filter: ${filterName}`);
}
const filter = this.filters.get(filterName.trim());
const args = [value, ...filterArgs.map(arg => this.parseValue(arg.trim(), context))];
value = filter(...args);
}
return value;
}
resolveVariable(path, context) {
const parts = path.split('.');
let value = context;
for (const part of parts) {
if (value && typeof value === 'object' && part in value) {
value = value[part];
} else {
return undefined;
}
}
return value;
}
parseArguments(argsStr, context) {
const args = [];
const argPattern = /(".*?"|'.*?'|\S+)/g;
let match;
while ((match = argPattern.exec(argsStr)) !== null) {
args.push(this.parseValue(match[1], context));
}
return args;
}
parseValue(valueStr, context) {
// Remove quotes if present
if ((valueStr.startsWith('"') && valueStr.endsWith('"')) ||
(valueStr.startsWith("'") && valueStr.endsWith("'"))) {
return valueStr.slice(1, -1);
}
// Check if it's a number
if (/^\d+(\.\d+)?$/.test(valueStr)) {
return parseFloat(valueStr);
}
// Check if it's a boolean
if (valueStr === 'true') return true;
if (valueStr === 'false') return false;
if (valueStr === 'null') return null;
// Try to resolve as variable
const resolved = this.resolveVariable(valueStr, context);
return resolved !== undefined ? resolved : valueStr;
}
parseContextExpression(expression, baseContext) {
const context = {};
const assignments = expression.split(/\s+/);
assignments.forEach(assignment => {
const [key, value] = assignment.split('=');
if (key && value) {
context[key] = this.parseValue(value, baseContext);
}
});
return context;
}
formatDate(date, format) {
const formatMap = {
'YYYY': date.getFullYear(),
'MM': String(date.getMonth() + 1).padStart(2, '0'),
'DD': String(date.getDate()).padStart(2, '0'),
'HH': String(date.getHours()).padStart(2, '0'),
'mm': String(date.getMinutes()).padStart(2, '0'),
'ss': String(date.getSeconds()).padStart(2, '0')
};
let formatted = format;
Object.entries(formatMap).forEach(([pattern, value]) => {
formatted = formatted.replace(new RegExp(pattern, 'g'), value);
});
return formatted;
}
async generate(templateName, data = {}, outputName = null) {
try {
const template = await this.loadTemplate(templateName);
const processed = await this.processTemplate(template, data);
if (outputName) {
const fs = require('fs').promises;
const path = require('path');
const outputPath = path.join(this.options.outputDirectory, `${outputName}.md`);
await fs.mkdir(this.options.outputDirectory, { recursive: true });
await fs.writeFile(outputPath, processed, this.options.encoding);
return { content: processed, outputPath, success: true };
}
return { content: processed, success: true };
} catch (error) {
return { error: error.message, success: false };
}
}
async generateFromDataFile(templateName, dataName, outputName = null) {
try {
const data = await this.loadData(dataName);
return await this.generate(templateName, data, outputName || dataName);
} catch (error) {
return { error: error.message, success: false };
}
}
async generateBatch(configurations) {
const results = [];
for (const config of configurations) {
const { template, data, output, dataFile } = config;
let result;
if (dataFile) {
result = await this.generateFromDataFile(template, dataFile, output);
} else {
result = await this.generate(template, data || {}, output);
}
results.push({
...config,
...result
});
}
return results;
}
}
// Usage example
async function demonstrateTemplateEngine() {
const engine = new MarkdownTemplateEngine({
templateDirectory: './templates',
dataDirectory: './data',
outputDirectory: './output',
cacheTemplates: true
});
// Register custom helper
engine.registerHelper('pluralize', (count, singular, plural) => {
return count === 1 ? singular : plural;
});
// Set global variables
engine.registerGlobal('site_name', 'My Documentation Site');
engine.registerGlobal('version', '2.0.0');
// Generate single document
const apiData = {
endpoints: [
{ method: 'GET', path: '/users', description: 'Get all users' },
{ method: 'POST', path: '/users', description: 'Create a user' },
{ method: 'GET', path: '/users/:id', description: 'Get user by ID' }
],
version: '1.0',
baseUrl: 'https://api.example.com'
};
const result = await engine.generate('api-documentation', apiData, 'api-docs');
if (result.success) {
console.log('Generated API documentation:', result.outputPath);
} else {
console.error('Generation failed:', result.error);
}
// Generate batch
const batchConfig = [
{
template: 'user-guide',
dataFile: 'user-guide-data',
output: 'user-guide'
},
{
template: 'api-reference',
dataFile: 'api-data',
output: 'api-reference'
},
{
template: 'changelog',
dataFile: 'releases',
output: 'changelog'
}
];
const batchResults = await engine.generateBatch(batchConfig);
console.log('Batch generation results:', batchResults);
}
module.exports = MarkdownTemplateEngine;
Advanced Data Source Integration
Connecting with various data sources for dynamic content generation:
// data-source-manager.js - Multi-source data integration
class DataSourceManager {
constructor(options = {}) {
this.options = {
cacheTimeout: options.cacheTimeout || 300000, // 5 minutes
retryAttempts: options.retryAttempts || 3,
retryDelay: options.retryDelay || 1000,
...options
};
this.dataSources = new Map();
this.cache = new Map();
this.registerDefaultSources();
}
registerDefaultSources() {
// File system data source
this.registerSource('file', {
load: async (path) => {
const fs = require('fs').promises;
const pathModule = require('path');
const ext = pathModule.extname(path).toLowerCase();
const content = await fs.readFile(path, 'utf-8');
switch (ext) {
case '.json':
return JSON.parse(content);
case '.yaml':
case '.yml':
const yaml = require('yaml');
return yaml.parse(content);
case '.csv':
return this.parseCSV(content);
case '.md':
return this.parseMarkdownWithFrontmatter(content);
default:
return content;
}
}
});
// HTTP/REST API data source
this.registerSource('http', {
load: async (url, options = {}) => {
const response = await fetch(url, {
method: options.method || 'GET',
headers: {
'Content-Type': 'application/json',
'User-Agent': 'Markdown-Generator/1.0',
...options.headers
},
body: options.body ? JSON.stringify(options.body) : undefined
});
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
const contentType = response.headers.get('content-type');
if (contentType && contentType.includes('application/json')) {
return await response.json();
} else {
return await response.text();
}
}
});
// Database data source
this.registerSource('database', {
load: async (connectionString, query) => {
// This would integrate with actual database drivers
// Example with PostgreSQL using pg library
/*
const { Client } = require('pg');
const client = new Client(connectionString);
await client.connect();
const result = await client.query(query);
await client.end();
return result.rows;
*/
throw new Error('Database integration requires specific database driver');
}
});
// Git repository data source
this.registerSource('git', {
load: async (repoPath, options = {}) => {
const { exec } = require('child_process');
const { promisify } = require('util');
const execAsync = promisify(exec);
const commands = {
commits: `git -C ${repoPath} log --pretty=format:'{"hash":"%H","author":"%an","date":"%ad","message":"%s"}' --date=iso`,
tags: `git -C ${repoPath} tag --sort=-version:refname`,
branches: `git -C ${repoPath} branch -a`,
contributors: `git -C ${repoPath} shortlog -sn`,
stats: `git -C ${repoPath} log --stat --pretty=format:''`
};
const command = commands[options.type] || commands.commits;
const { stdout } = await execAsync(command);
switch (options.type) {
case 'commits':
return stdout.split('\n')
.filter(line => line.trim())
.map(line => JSON.parse(line));
case 'tags':
case 'branches':
return stdout.split('\n').filter(line => line.trim());
case 'contributors':
return stdout.split('\n')
.filter(line => line.trim())
.map(line => {
const match = line.match(/^\s*(\d+)\s+(.+)$/);
return match ? {
commits: parseInt(match[1]),
name: match[2]
} : null;
})
.filter(Boolean);
default:
return stdout;
}
}
});
// Environment variables data source
this.registerSource('env', {
load: async (prefix = '') => {
const env = {};
Object.keys(process.env).forEach(key => {
if (!prefix || key.startsWith(prefix)) {
const cleanKey = prefix ? key.substring(prefix.length) : key;
env[cleanKey.toLowerCase()] = process.env[key];
}
});
return env;
}
});
// Package.json data source
this.registerSource('package', {
load: async (packagePath = './package.json') => {
const fs = require('fs').promises;
const content = await fs.readFile(packagePath, 'utf-8');
return JSON.parse(content);
}
});
}
registerSource(name, source) {
this.dataSources.set(name, source);
}
async loadData(sourceType, sourceIdentifier, options = {}) {
const cacheKey = `${sourceType}:${JSON.stringify(sourceIdentifier)}:${JSON.stringify(options)}`;
// Check cache
if (this.cache.has(cacheKey)) {
const cached = this.cache.get(cacheKey);
if (Date.now() - cached.timestamp < this.options.cacheTimeout) {
return cached.data;
}
}
// Load from source
if (!this.dataSources.has(sourceType)) {
throw new Error(`Unknown data source type: ${sourceType}`);
}
const source = this.dataSources.get(sourceType);
let attempts = 0;
while (attempts < this.options.retryAttempts) {
try {
const data = await source.load(sourceIdentifier, options);
// Cache the result
this.cache.set(cacheKey, {
data,
timestamp: Date.now()
});
return data;
} catch (error) {
attempts++;
if (attempts >= this.options.retryAttempts) {
throw new Error(`Failed to load data after ${attempts} attempts: ${error.message}`);
}
await new Promise(resolve => setTimeout(resolve, this.options.retryDelay * attempts));
}
}
}
async loadMultiple(sources) {
const results = {};
const promises = sources.map(async ({ name, type, identifier, options }) => {
try {
const data = await this.loadData(type, identifier, options);
results[name] = data;
} catch (error) {
console.error(`Failed to load ${name}:`, error.message);
results[name] = null;
}
});
await Promise.all(promises);
return results;
}
parseCSV(content) {
const lines = content.split('\n').filter(line => line.trim());
if (lines.length === 0) return [];
const headers = lines[0].split(',').map(h => h.trim().replace(/"/g, ''));
const rows = lines.slice(1).map(line => {
const values = line.split(',').map(v => v.trim().replace(/"/g, ''));
const row = {};
headers.forEach((header, index) => {
row[header] = values[index] || '';
});
return row;
});
return rows;
}
parseMarkdownWithFrontmatter(content) {
const frontmatterRegex = /^---\s*\n([\s\S]*?)\n---\s*\n([\s\S]*)$/;
const match = content.match(frontmatterRegex);
if (match) {
const yaml = require('yaml');
const frontmatter = yaml.parse(match[1]);
const body = match[2];
return {
frontmatter,
content: body,
raw: content
};
}
return {
frontmatter: {},
content: content,
raw: content
};
}
clearCache(pattern = null) {
if (pattern) {
const regex = new RegExp(pattern);
for (const key of this.cache.keys()) {
if (regex.test(key)) {
this.cache.delete(key);
}
}
} else {
this.cache.clear();
}
}
getCacheStats() {
return {
totalEntries: this.cache.size,
memoryUsage: JSON.stringify([...this.cache.values()]).length,
oldestEntry: Math.min(...[...this.cache.values()].map(entry => entry.timestamp)),
newestEntry: Math.max(...[...this.cache.values()].map(entry => entry.timestamp))
};
}
}
// Integration example
async function demonstrateDataIntegration() {
const dataManager = new DataSourceManager({
cacheTimeout: 300000, // 5 minutes
retryAttempts: 3
});
// Load multiple data sources
const dataSources = [
{
name: 'package_info',
type: 'package',
identifier: './package.json'
},
{
name: 'api_data',
type: 'http',
identifier: 'https://api.github.com/repos/user/repo',
options: {
headers: {
'Authorization': 'token YOUR_TOKEN'
}
}
},
{
name: 'config',
type: 'file',
identifier: './config/app.yaml'
},
{
name: 'git_commits',
type: 'git',
identifier: '.',
options: { type: 'commits' }
},
{
name: 'environment',
type: 'env',
identifier: 'APP_'
}
];
const allData = await dataManager.loadMultiple(dataSources);
console.log('Loaded data sources:', Object.keys(allData));
// Use with template engine
const templateEngine = new MarkdownTemplateEngine();
const generationResult = await templateEngine.generate(
'comprehensive-documentation',
allData,
'generated-docs'
);
if (generationResult.success) {
console.log('Documentation generated successfully');
}
}
module.exports = DataSourceManager;
Automated Documentation Generation
API Documentation Generator
Creating comprehensive API documentation from code and configuration:
// api-doc-generator.js - Automated API documentation system
class APIDocumentationGenerator {
constructor(options = {}) {
this.options = {
sourceDirectory: options.sourceDirectory || './src',
outputDirectory: options.outputDirectory || './docs/api',
templateDirectory: options.templateDirectory || './templates/api',
includePrivate: options.includePrivate || false,
includeInternal: options.includeInternal || false,
generateExamples: options.generateExamples !== false,
includeSchemas: options.includeSchemas !== false,
format: options.format || 'markdown',
groupBy: options.groupBy || 'module', // module, tag, path
...options
};
this.templateEngine = new MarkdownTemplateEngine({
templateDirectory: this.options.templateDirectory
});
this.dataManager = new DataSourceManager();
this.parsers = new Map();
this.generators = new Map();
this.setupDefaultParsers();
this.setupDefaultGenerators();
}
setupDefaultParsers() {
// OpenAPI/Swagger parser
this.registerParser('openapi', {
pattern: /\.(yaml|yml|json)$/,
parse: async (filePath) => {
const data = await this.dataManager.loadData('file', filePath);
if (this.isOpenAPISpec(data)) {
return this.parseOpenAPISpec(data);
}
return null;
}
});
// JSDoc parser
this.registerParser('jsdoc', {
pattern: /\.(js|ts|jsx|tsx)$/,
parse: async (filePath) => {
const fs = require('fs').promises;
const content = await fs.readFile(filePath, 'utf-8');
return this.parseJSDocComments(content, filePath);
}
});
// Python docstring parser
this.registerParser('python', {
pattern: /\.py$/,
parse: async (filePath) => {
const fs = require('fs').promises;
const content = await fs.readFile(filePath, 'utf-8');
return this.parsePythonDocstrings(content, filePath);
}
});
// Go doc parser
this.registerParser('go', {
pattern: /\.go$/,
parse: async (filePath) => {
const fs = require('fs').promises;
const content = await fs.readFile(filePath, 'utf-8');
return this.parseGoComments(content, filePath);
}
});
}
setupDefaultGenerators() {
// Endpoint documentation generator
this.registerGenerator('endpoints', {
template: 'endpoint-list',
process: (apiData) => {
const endpoints = [];
if (apiData.paths) {
Object.entries(apiData.paths).forEach(([path, methods]) => {
Object.entries(methods).forEach(([method, spec]) => {
endpoints.push({
method: method.toUpperCase(),
path,
summary: spec.summary || '',
description: spec.description || '',
parameters: spec.parameters || [],
responses: spec.responses || {},
tags: spec.tags || [],
operationId: spec.operationId,
deprecated: spec.deprecated || false,
examples: this.generateEndpointExamples(method, path, spec)
});
});
});
}
return { endpoints: this.groupEndpoints(endpoints) };
}
});
// Schema documentation generator
this.registerGenerator('schemas', {
template: 'schema-definitions',
process: (apiData) => {
const schemas = {};
if (apiData.components && apiData.components.schemas) {
Object.entries(apiData.components.schemas).forEach(([name, schema]) => {
schemas[name] = {
name,
type: schema.type || 'object',
description: schema.description || '',
properties: this.processSchemaProperties(schema.properties || {}),
required: schema.required || [],
example: schema.example || this.generateSchemaExample(schema)
};
});
}
return { schemas };
}
});
// Authentication documentation generator
this.registerGenerator('authentication', {
template: 'auth-methods',
process: (apiData) => {
const authMethods = [];
if (apiData.components && apiData.components.securitySchemes) {
Object.entries(apiData.components.securitySchemes).forEach(([name, scheme]) => {
authMethods.push({
name,
type: scheme.type,
description: scheme.description || '',
scheme: scheme.scheme,
bearerFormat: scheme.bearerFormat,
flows: scheme.flows,
openIdConnectUrl: scheme.openIdConnectUrl,
examples: this.generateAuthExamples(scheme)
});
});
}
return { authMethods };
}
});
// Error code documentation generator
this.registerGenerator('errors', {
template: 'error-codes',
process: (apiData) => {
const errorCodes = new Set();
const errors = {};
// Extract error codes from all endpoint responses
if (apiData.paths) {
Object.values(apiData.paths).forEach(methods => {
Object.values(methods).forEach(spec => {
if (spec.responses) {
Object.entries(spec.responses).forEach(([code, response]) => {
if (parseInt(code) >= 400) {
errorCodes.add(code);
errors[code] = {
code,
description: response.description || '',
schema: response.content &&
response.content['application/json'] &&
response.content['application/json'].schema,
examples: response.examples || {}
};
}
});
}
});
});
}
return { errors: Object.values(errors) };
}
});
}
registerParser(name, parser) {
this.parsers.set(name, parser);
}
registerGenerator(name, generator) {
this.generators.set(name, generator);
}
async generateDocumentation(configPath = null) {
let config = this.options;
if (configPath) {
const configData = await this.dataManager.loadData('file', configPath);
config = { ...config, ...configData };
}
console.log('Starting API documentation generation...');
// Discover and parse API specifications
const apiData = await this.discoverAndParseAPIs();
// Generate documentation sections
const results = [];
for (const [generatorName, generator] of this.generators) {
console.log(`Generating ${generatorName} documentation...`);
try {
const processedData = generator.process(apiData);
const result = await this.templateEngine.generate(
generator.template,
{
...processedData,
apiInfo: apiData.info || {},
generated_at: new Date().toISOString(),
generator_version: '1.0.0'
},
generatorName
);
results.push({
section: generatorName,
success: result.success,
outputPath: result.outputPath,
error: result.error
});
} catch (error) {
console.error(`Failed to generate ${generatorName}:`, error.message);
results.push({
section: generatorName,
success: false,
error: error.message
});
}
}
// Generate index/overview page
await this.generateIndexPage(apiData, results);
return results;
}
async discoverAndParseAPIs() {
const fs = require('fs').promises;
const path = require('path');
const glob = require('glob');
const { promisify } = require('util');
const globAsync = promisify(glob);
const allApiData = {
info: {},
paths: {},
components: { schemas: {}, securitySchemes: {} },
tags: []
};
// Find all potential API definition files
const patterns = [
'**/*.yaml',
'**/*.yml',
'**/*.json',
'**/*.js',
'**/*.ts',
'**/*.py',
'**/*.go'
];
for (const pattern of patterns) {
const files = await globAsync(path.join(this.options.sourceDirectory, pattern), {
ignore: ['**/node_modules/**', '**/vendor/**', '**/.git/**']
});
for (const file of files) {
for (const [parserName, parser] of this.parsers) {
if (parser.pattern.test(file)) {
try {
const parsed = await parser.parse(file);
if (parsed) {
this.mergeApiData(allApiData, parsed);
}
} catch (error) {
console.warn(`Failed to parse ${file} with ${parserName}: ${error.message}`);
}
}
}
}
}
return allApiData;
}
isOpenAPISpec(data) {
return data && (data.openapi || data.swagger) && data.paths;
}
parseOpenAPISpec(spec) {
return {
info: spec.info || {},
paths: spec.paths || {},
components: spec.components || {},
tags: spec.tags || [],
servers: spec.servers || []
};
}
parseJSDocComments(content, filePath) {
const apiData = { paths: {}, components: { schemas: {} } };
// Extract JSDoc comments
const jsdocPattern = /\/\*\*\s*([\s\S]*?)\s*\*\//g;
let match;
while ((match = jsdocPattern.exec(content)) !== null) {
const comment = match[1];
// Look for API endpoint definitions
const routeMatch = comment.match(/@route\s+(\w+)\s+([^\n]+)/);
const descMatch = comment.match(/@description\s+([^\n]+)/);
const paramMatches = [...comment.matchAll(/@param\s+{([^}]+)}\s+(\w+)\s+([^\n]+)/g)];
const returnMatch = comment.match(/@returns\s+{([^}]+)}\s+([^\n]+)/);
if (routeMatch) {
const [, method, path] = routeMatch;
const description = descMatch ? descMatch[1] : '';
if (!apiData.paths[path]) {
apiData.paths[path] = {};
}
apiData.paths[path][method.toLowerCase()] = {
summary: description,
description: description,
parameters: paramMatches.map(([, type, name, desc]) => ({
name,
in: 'query', // Default, could be enhanced
description: desc,
schema: { type: this.mapJSTypeToOpenAPI(type) }
})),
responses: {
'200': {
description: returnMatch ? returnMatch[2] : 'Success',
content: {
'application/json': {
schema: returnMatch ?
{ type: this.mapJSTypeToOpenAPI(returnMatch[1]) } :
{ type: 'object' }
}
}
}
}
};
}
}
return Object.keys(apiData.paths).length > 0 ? apiData : null;
}
parsePythonDocstrings(content, filePath) {
// Basic Python docstring parsing
const apiData = { paths: {}, components: { schemas: {} } };
// Look for Flask/FastAPI decorators and docstrings
const routePattern = /@app\.(get|post|put|delete|patch)\(['"]([^'"]+)['"]\)/g;
let match;
while ((match = routePattern.exec(content)) !== null) {
const [fullMatch, method, path] = match;
const methodIndex = match.index + fullMatch.length;
// Look for the function and its docstring
const funcPattern = /def\s+(\w+)\s*\([^)]*\):\s*"""([^"]+)"""/;
const funcMatch = content.substring(methodIndex).match(funcPattern);
if (funcMatch) {
const [, funcName, docstring] = funcMatch;
if (!apiData.paths[path]) {
apiData.paths[path] = {};
}
apiData.paths[path][method] = {
summary: funcName.replace(/_/g, ' '),
description: docstring.trim(),
operationId: funcName,
responses: {
'200': {
description: 'Success',
content: {
'application/json': {
schema: { type: 'object' }
}
}
}
}
};
}
}
return Object.keys(apiData.paths).length > 0 ? apiData : null;
}
parseGoComments(content, filePath) {
// Basic Go comment parsing
const apiData = { paths: {}, components: { schemas: {} } };
// Look for HTTP handlers with comments
const handlerPattern = /\/\/\s*@(\w+)\s+([^\n]+)\n.*?func\s+(\w+)/g;
let match;
while ((match = handlerPattern.exec(content)) !== null) {
const [, annotation, details, funcName] = match;
if (annotation.toLowerCase() === 'route') {
const routeMatch = details.match(/(\w+)\s+([^\s]+)/);
if (routeMatch) {
const [, method, path] = routeMatch;
if (!apiData.paths[path]) {
apiData.paths[path] = {};
}
apiData.paths[path][method.toLowerCase()] = {
summary: funcName,
description: `Handler function: ${funcName}`,
operationId: funcName,
responses: {
'200': {
description: 'Success'
}
}
};
}
}
}
return Object.keys(apiData.paths).length > 0 ? apiData : null;
}
mergeApiData(target, source) {
// Merge paths
Object.entries(source.paths || {}).forEach(([path, methods]) => {
if (!target.paths[path]) {
target.paths[path] = {};
}
Object.assign(target.paths[path], methods);
});
// Merge components
if (source.components) {
if (source.components.schemas) {
Object.assign(target.components.schemas, source.components.schemas);
}
if (source.components.securitySchemes) {
Object.assign(target.components.securitySchemes, source.components.securitySchemes);
}
}
// Merge info (prefer first found)
if (source.info && !target.info.title) {
target.info = source.info;
}
// Merge tags
if (source.tags) {
target.tags = [...target.tags, ...source.tags];
}
}
mapJSTypeToOpenAPI(jsType) {
const typeMap = {
'string': 'string',
'number': 'number',
'boolean': 'boolean',
'object': 'object',
'array': 'array',
'Array': 'array',
'Object': 'object',
'String': 'string',
'Number': 'number',
'Boolean': 'boolean'
};
return typeMap[jsType] || 'string';
}
groupEndpoints(endpoints) {
const grouped = {};
endpoints.forEach(endpoint => {
let groupKey;
switch (this.options.groupBy) {
case 'tag':
groupKey = endpoint.tags.length > 0 ? endpoint.tags[0] : 'untagged';
break;
case 'path':
groupKey = endpoint.path.split('/')[1] || 'root';
break;
case 'method':
groupKey = endpoint.method;
break;
default:
groupKey = 'api';
}
if (!grouped[groupKey]) {
grouped[groupKey] = [];
}
grouped[groupKey].push(endpoint);
});
return grouped;
}
generateEndpointExamples(method, path, spec) {
const examples = {};
// Generate curl example
let curl = `curl -X ${method.toUpperCase()} "${path}"`;
if (spec.parameters) {
const queryParams = spec.parameters
.filter(p => p.in === 'query')
.map(p => `${p.name}=example_value`)
.join('&');
if (queryParams) {
curl += `?${queryParams}`;
}
}
if (['POST', 'PUT', 'PATCH'].includes(method.toUpperCase())) {
curl += ' -H "Content-Type: application/json"';
curl += ' -d \'{"example": "data"}\'';
}
examples.curl = curl;
// Generate JavaScript example
examples.javascript = this.generateJavaScriptExample(method, path, spec);
// Generate Python example
examples.python = this.generatePythonExample(method, path, spec);
return examples;
}
generateJavaScriptExample(method, path, spec) {
let js = `const response = await fetch('${path}', {\n`;
js += ` method: '${method.toUpperCase()}',\n`;
js += ` headers: {\n`;
js += ` 'Content-Type': 'application/json',\n`;
js += ` },\n`;
if (['POST', 'PUT', 'PATCH'].includes(method.toUpperCase())) {
js += ` body: JSON.stringify({\n`;
js += ` example: 'data'\n`;
js += ` })\n`;
}
js += `});\n\n`;
js += `const data = await response.json();\n`;
js += `console.log(data);`;
return js;
}
generatePythonExample(method, path, spec) {
let python = `import requests\n\n`;
if (['POST', 'PUT', 'PATCH'].includes(method.toUpperCase())) {
python += `response = requests.${method.toLowerCase()}(\n`;
python += ` '${path}',\n`;
python += ` json={'example': 'data'},\n`;
python += ` headers={'Content-Type': 'application/json'}\n`;
python += `)\n`;
} else {
python += `response = requests.${method.toLowerCase()}('${path}')\n`;
}
python += `\nif response.status_code == 200:\n`;
python += ` data = response.json()\n`;
python += ` print(data)\n`;
python += `else:\n`;
python += ` print(f"Error: {response.status_code}")`;
return python;
}
processSchemaProperties(properties) {
const processed = {};
Object.entries(properties).forEach(([name, prop]) => {
processed[name] = {
type: prop.type || 'string',
description: prop.description || '',
example: prop.example || this.generatePropertyExample(prop),
required: false // Will be set by parent schema
};
});
return processed;
}
generateSchemaExample(schema) {
if (schema.example) return schema.example;
const example = {};
if (schema.properties) {
Object.entries(schema.properties).forEach(([name, prop]) => {
example[name] = this.generatePropertyExample(prop);
});
}
return example;
}
generatePropertyExample(property) {
switch (property.type) {
case 'string':
return property.format === 'email' ? '[email protected]' :
property.format === 'date' ? '2023-12-25' :
property.format === 'uuid' ? '123e4567-e89b-12d3-a456-426614174000' :
'example string';
case 'number':
case 'integer':
return 42;
case 'boolean':
return true;
case 'array':
return ['example item'];
case 'object':
return { example: 'value' };
default:
return 'example';
}
}
generateAuthExamples(scheme) {
const examples = {};
switch (scheme.type) {
case 'apiKey':
if (scheme.in === 'header') {
examples.curl = `curl -H "${scheme.name}: YOUR_API_KEY" https://api.example.com/endpoint`;
examples.javascript = `fetch('https://api.example.com/endpoint', {\n headers: {\n '${scheme.name}': 'YOUR_API_KEY'\n }\n})`;
}
break;
case 'http':
if (scheme.scheme === 'bearer') {
examples.curl = `curl -H "Authorization: Bearer YOUR_TOKEN" https://api.example.com/endpoint`;
examples.javascript = `fetch('https://api.example.com/endpoint', {\n headers: {\n 'Authorization': 'Bearer YOUR_TOKEN'\n }\n})`;
}
break;
case 'oauth2':
examples.description = 'OAuth2 authentication flow required';
break;
}
return examples;
}
async generateIndexPage(apiData, results) {
const indexData = {
title: apiData.info.title || 'API Documentation',
version: apiData.info.version || '1.0.0',
description: apiData.info.description || '',
sections: results.filter(r => r.success),
generationTime: new Date().toISOString(),
totalEndpoints: Object.keys(apiData.paths || {}).length,
totalSchemas: Object.keys(apiData.components?.schemas || {}).length
};
return await this.templateEngine.generate('api-index', indexData, 'index');
}
}
// Usage example
async function demonstrateAPIGeneration() {
const generator = new APIDocumentationGenerator({
sourceDirectory: './src/api',
outputDirectory: './docs/api',
templateDirectory: './templates/api',
includeExamples: true,
groupBy: 'tag'
});
const results = await generator.generateDocumentation();
console.log('API Documentation Generation Results:');
results.forEach(result => {
console.log(`${result.section}: ${result.success ? 'Success' : 'Failed'}`);
if (result.error) {
console.error(` Error: ${result.error}`);
}
});
}
module.exports = APIDocumentationGenerator;
Integration with Modern Development Workflows
Dynamic Markdown content generation integrates seamlessly with comprehensive documentation ecosystems. When combined with automated workflow systems and pipeline management, dynamic generation becomes part of continuous integration processes, ensuring that documentation updates automatically whenever code changes occur.
For sophisticated content architectures, dynamic generation works effectively with content validation and quality assurance systems to ensure that generated content meets organizational standards for accuracy, formatting, and accessibility, creating documentation pipelines that maintain both automation and quality.
When building scalable documentation platforms, dynamic content generation complements version control and Git integration features by automatically committing generated documentation updates, managing documentation versioning in sync with code releases, and ensuring that documentation branches remain synchronized with development workflows.
Advanced Template Systems and Customization
Complex Template Inheritance and Composition
Building sophisticated template hierarchies for maintainable documentation systems:
// advanced-template-system.js - Sophisticated template management
class AdvancedTemplateSystem extends MarkdownTemplateEngine {
constructor(options = {}) {
super(options);
this.layouts = new Map();
this.mixins = new Map();
this.components = new Map();
this.themes = new Map();
this.preprocessors = [];
this.postprocessors = [];
this.setupAdvancedFeatures();
}
setupAdvancedFeatures() {
// Layout system
this.registerHelper('layout', (layoutName, content) => {
return this.applyLayout(layoutName, content);
});
// Component system
this.registerHelper('component', (componentName, props = {}) => {
return this.renderComponent(componentName, props);
});
// Mixin system
this.registerHelper('mixin', (mixinName, ...args) => {
return this.applyMixin(mixinName, ...args);
});
// Conditional blocks
this.registerHelper('ifblock', (condition, ifContent, elseContent = '') => {
return condition ? ifContent : elseContent;
});
// Switch/case helper
this.registerHelper('switch', (value, cases) => {
return cases[value] || cases.default || '';
});
// Template inheritance
this.registerHelper('extends', (parentTemplate) => {
return `<!-- EXTENDS: ${parentTemplate} -->`;
});
// Block definitions
this.registerHelper('block', (blockName, defaultContent = '') => {
return `<!-- BLOCK: ${blockName} -->${defaultContent}<!-- /BLOCK: ${blockName} -->`;
});
// Include with fallback
this.registerHelper('include_safe', async (templateName, fallbackContent = '') => {
try {
return await this.loadTemplate(templateName);
} catch {
return fallbackContent;
}
});
}
async registerLayout(name, template) {
this.layouts.set(name, template);
}
async registerComponent(name, template) {
this.components.set(name, template);
}
registerMixin(name, mixinFunction) {
this.mixins.set(name, mixinFunction);
}
addPreprocessor(processor) {
this.preprocessors.push(processor);
}
addPostprocessor(processor) {
this.postprocessors.push(processor);
}
async processTemplate(template, context = {}) {
let processed = template;
// Apply preprocessors
for (const processor of this.preprocessors) {
processed = await processor(processed, context);
}
// Handle template inheritance
processed = await this.processInheritance(processed, context);
// Apply standard template processing
processed = await super.processTemplate(processed, context);
// Apply postprocessors
for (const processor of this.postprocessors) {
processed = await processor(processed, context);
}
return processed;
}
async processInheritance(template, context) {
const extendsMatch = template.match(/<!--\s*EXTENDS:\s*(\w+)\s*-->/);
if (extendsMatch) {
const parentName = extendsMatch[1];
const parentTemplate = await this.loadTemplate(parentName);
// Extract blocks from child template
const blocks = this.extractBlocks(template);
// Replace blocks in parent template
let processed = parentTemplate;
Object.entries(blocks).forEach(([blockName, blockContent]) => {
const blockPattern = new RegExp(
`<!--\\s*BLOCK:\\s*${blockName}\\s*-->([\\s\\S]*?)<!--\\s*/BLOCK:\\s*${blockName}\\s*-->`,
'g'
);
processed = processed.replace(blockPattern, `<!-- BLOCK: ${blockName} -->${blockContent}<!-- /BLOCK: ${blockName} -->`);
});
return processed;
}
return template;
}
extractBlocks(template) {
const blocks = {};
const blockPattern = /<!--\s*BLOCK:\s*(\w+)\s*-->([\s\S]*?)<!--\s*\/BLOCK:\s*\1\s*-->/g;
let match;
while ((match = blockPattern.exec(template)) !== null) {
const [, blockName, blockContent] = match;
blocks[blockName] = blockContent.trim();
}
return blocks;
}
async applyLayout(layoutName, content) {
if (!this.layouts.has(layoutName)) {
const layout = await this.loadTemplate(`layouts/${layoutName}`);
this.layouts.set(layoutName, layout);
}
const layout = this.layouts.get(layoutName);
return layout.replace(/{{content}}/g, content);
}
async renderComponent(componentName, props) {
if (!this.components.has(componentName)) {
const component = await this.loadTemplate(`components/${componentName}`);
this.components.set(componentName, component);
}
const component = this.components.get(componentName);
return await this.processTemplate(component, props);
}
applyMixin(mixinName, ...args) {
if (!this.mixins.has(mixinName)) {
throw new Error(`Unknown mixin: ${mixinName}`);
}
const mixin = this.mixins.get(mixinName);
return mixin(...args);
}
// Advanced data transformation helpers
setupDataTransformers() {
this.registerHelper('transform', (data, transformer) => {
const transformers = {
'group_by': (items, key) => {
const grouped = {};
items.forEach(item => {
const groupKey = item[key];
if (!grouped[groupKey]) {
grouped[groupKey] = [];
}
grouped[groupKey].push(item);
});
return grouped;
},
'sort_by': (items, key, order = 'asc') => {
return items.slice().sort((a, b) => {
const valueA = a[key];
const valueB = b[key];
let comparison = 0;
if (valueA > valueB) comparison = 1;
else if (valueA < valueB) comparison = -1;
return order === 'desc' ? -comparison : comparison;
});
},
'filter_by': (items, key, value) => {
return items.filter(item => item[key] === value);
},
'map': (items, key) => {
return items.map(item => item[key]);
},
'flatten': (nestedArray) => {
return nestedArray.flat(Infinity);
},
'unique': (items) => {
return [...new Set(items)];
},
'chunk': (items, size) => {
const chunks = [];
for (let i = 0; i < items.length; i += size) {
chunks.push(items.slice(i, i + size));
}
return chunks;
}
};
const transformerFn = transformers[transformer];
return transformerFn ? transformerFn(data) : data;
});
// Multi-step transformation pipeline
this.registerHelper('pipeline', (data, ...transformers) => {
return transformers.reduce((result, transformer) => {
if (typeof transformer === 'string') {
return this.helpers.get('transform')(result, transformer);
} else if (typeof transformer === 'object') {
const { name, args = [] } = transformer;
return this.helpers.get('transform')(result, name, ...args);
}
return result;
}, data);
});
}
// Theme support
async loadTheme(themeName) {
const fs = require('fs').promises;
const path = require('path');
const themeDir = path.join(this.options.templateDirectory, 'themes', themeName);
const themeConfigPath = path.join(themeDir, 'theme.json');
try {
const themeConfig = JSON.parse(await fs.readFile(themeConfigPath, 'utf-8'));
// Load theme templates
const themeTemplates = {};
if (themeConfig.templates) {
for (const templateName of Object.keys(themeConfig.templates)) {
const templatePath = path.join(themeDir, `${templateName}.md`);
themeTemplates[templateName] = await fs.readFile(templatePath, 'utf-8');
}
}
// Load theme partials
const themePartials = {};
if (themeConfig.partials) {
for (const partialName of Object.keys(themeConfig.partials)) {
const partialPath = path.join(themeDir, 'partials', `${partialName}.md`);
themePartials[partialName] = await fs.readFile(partialPath, 'utf-8');
}
}
const theme = {
config: themeConfig,
templates: themeTemplates,
partials: themePartials
};
this.themes.set(themeName, theme);
return theme;
} catch (error) {
throw new Error(`Failed to load theme ${themeName}: ${error.message}`);
}
}
async applyTheme(themeName) {
const theme = await this.loadTheme(themeName);
// Override templates with theme templates
Object.entries(theme.templates).forEach(([name, template]) => {
this.templateCache.set(name, template);
});
// Override partials with theme partials
Object.entries(theme.partials).forEach(([name, partial]) => {
this.partialCache.set(name, partial);
});
// Apply theme configuration
if (theme.config.globals) {
Object.entries(theme.config.globals).forEach(([name, value]) => {
this.registerGlobal(name, value);
});
}
if (theme.config.helpers) {
Object.entries(theme.config.helpers).forEach(([name, helperConfig]) => {
// Custom helper registration logic
this.registerThemeHelper(name, helperConfig);
});
}
}
registerThemeHelper(name, config) {
// Implementation would depend on helper configuration format
// This is a placeholder for theme-specific helper registration
console.log(`Registering theme helper: ${name}`, config);
}
// Conditional compilation
setupConditionalCompilation() {
this.addPreprocessor(async (template, context) => {
// Process conditional blocks: {{#ifdef VARIABLE}}...{{/ifdef}}
const ifdefPattern = /{{#ifdef\s+(\w+)}}([\s\S]*?){{\/ifdef}}/g;
return template.replace(ifdefPattern, (match, variable, content) => {
return context[variable] !== undefined ? content : '';
});
});
this.addPreprocessor(async (template, context) => {
// Process environment-specific blocks: {{#env production}}...{{/env}}
const envPattern = /{{#env\s+(\w+)}}([\s\S]*?){{\/env}}/g;
return template.replace(envPattern, (match, env, content) => {
return process.env.NODE_ENV === env ? content : '';
});
});
}
// Advanced caching strategies
setupAdvancedCaching() {
// Template dependency tracking
this.templateDependencies = new Map();
// Invalidation strategies
this.addCacheInvalidationRule('time', 3600000); // 1 hour
this.addCacheInvalidationRule('file_change', true);
this.addCacheInvalidationRule('data_change', true);
}
addCacheInvalidationRule(type, value) {
// Implementation for cache invalidation rules
console.log(`Added cache invalidation rule: ${type}`, value);
}
}
// Specialized generators using advanced template system
class DocumentationSiteGenerator extends AdvancedTemplateSystem {
constructor(options = {}) {
super(options);
this.setupDocumentationSpecificFeatures();
}
setupDocumentationSpecificFeatures() {
// Documentation-specific helpers
this.registerHelper('toc', (content, maxDepth = 3) => {
return this.generateTableOfContents(content, maxDepth);
});
this.registerHelper('breadcrumb', (currentPath, siteStructure) => {
return this.generateBreadcrumb(currentPath, siteStructure);
});
this.registerHelper('nav', (currentPage, navigation) => {
return this.generateNavigation(currentPage, navigation);
});
this.registerHelper('search_index', (pages) => {
return this.generateSearchIndex(pages);
});
this.registerHelper('related_pages', (currentPage, allPages, limit = 5) => {
return this.findRelatedPages(currentPage, allPages, limit);
});
// Setup documentation-specific mixins
this.setupDocumentationMixins();
}
setupDocumentationMixins() {
this.registerMixin('code_example', (code, language, title = '') => {
return `${title ? `**${title}**\n\n` : ''}\`\`\`${language}\n${code}\n\`\`\``;
});
this.registerMixin('api_endpoint', (method, path, description, params = []) => {
let output = `### ${method.toUpperCase()} ${path}\n\n${description}\n\n`;
if (params.length > 0) {
output += `**Parameters:**\n\n`;
params.forEach(param => {
output += `- **${param.name}** (${param.type}): ${param.description}\n`;
});
}
return output;
});
this.registerMixin('warning_box', (message, type = 'warning') => {
const icons = {
warning: '⚠️',
info: 'ℹ️',
danger: '🚨',
success: '✅'
};
return `> ${icons[type] || icons.warning} **${type.toUpperCase()}**\n> \n> ${message}`;
});
}
generateTableOfContents(content, maxDepth) {
const headings = content.match(/^#{1,6}\s+.+$/gm) || [];
let toc = '';
headings.forEach(heading => {
const level = heading.match(/^#+/)[0].length;
if (level <= maxDepth) {
const text = heading.replace(/^#+\s+/, '');
const anchor = text.toLowerCase().replace(/[^\w\s-]/g, '').replace(/\s+/g, '-');
const indent = ' '.repeat(level - 1);
toc += `${indent}- [${text}](#${anchor})\n`;
}
});
return toc;
}
generateBreadcrumb(currentPath, siteStructure) {
const pathParts = currentPath.split('/').filter(part => part);
const breadcrumbs = ['Home'];
let currentLevel = siteStructure;
pathParts.forEach(part => {
if (currentLevel[part]) {
breadcrumbs.push(currentLevel[part].title || part);
currentLevel = currentLevel[part].children || {};
}
});
return breadcrumbs.join(' > ');
}
generateNavigation(currentPage, navigation) {
let nav = '';
Object.entries(navigation).forEach(([key, item]) => {
const isActive = currentPage === key;
const activeClass = isActive ? ' (current)' : '';
nav += `- [${item.title}](${item.path})${activeClass}\n`;
if (item.children && isActive) {
Object.entries(item.children).forEach(([childKey, child]) => {
nav += ` - [${child.title}](${child.path})\n`;
});
}
});
return nav;
}
generateSearchIndex(pages) {
const searchIndex = pages.map(page => ({
title: page.title,
path: page.path,
excerpt: page.content.substring(0, 200),
keywords: page.keywords || [],
section: page.section || 'general'
}));
return JSON.stringify(searchIndex, null, 2);
}
findRelatedPages(currentPage, allPages, limit) {
// Simple related pages algorithm based on shared keywords
const currentKeywords = currentPage.keywords || [];
const scored = allPages
.filter(page => page.path !== currentPage.path)
.map(page => {
const sharedKeywords = (page.keywords || [])
.filter(keyword => currentKeywords.includes(keyword));
return {
...page,
score: sharedKeywords.length
};
})
.filter(page => page.score > 0)
.sort((a, b) => b.score - a.score)
.slice(0, limit);
return scored;
}
}
module.exports = { AdvancedTemplateSystem, DocumentationSiteGenerator };
Performance Optimization and Scalability
High-Performance Generation Pipeline
Optimizing dynamic content generation for large-scale documentation systems:
// performance-optimized-generator.js - Scalable content generation
class PerformanceOptimizedGenerator {
constructor(options = {}) {
this.options = {
maxConcurrency: options.maxConcurrency || require('os').cpus().length,
cacheStrategy: options.cacheStrategy || 'memory', // memory, file, redis
batchSize: options.batchSize || 50,
enableProfiling: options.enableProfiling || false,
enableMetrics: options.enableMetrics || false,
compressionEnabled: options.compressionEnabled || true,
incrementalGeneration: options.incrementalGeneration !== false,
...options
};
this.templateEngine = new AdvancedTemplateSystem(options);
this.cache = this.setupCache();
this.metrics = new Map();
this.dependencyGraph = new Map();
this.generationQueue = [];
this.workers = [];
this.setupPerformanceOptimizations();
}
setupCache() {
switch (this.options.cacheStrategy) {
case 'memory':
return new Map();
case 'file':
return new FileCache(this.options.cacheDirectory);
case 'redis':
return new RedisCache(this.options.redisConfig);
default:
return new Map();
}
}
setupPerformanceOptimizations() {
// Template compilation caching
this.compiledTemplates = new Map();
// Dependency tracking
this.templateDependencies = new Map();
this.dataDependencies = new Map();
// Performance monitoring
if (this.options.enableProfiling) {
this.setupProfiling();
}
if (this.options.enableMetrics) {
this.setupMetrics();
}
// Worker pool setup
this.setupWorkerPool();
}
setupWorkerPool() {
const { Worker } = require('worker_threads');
for (let i = 0; i < this.options.maxConcurrency; i++) {
const worker = new Worker(`
const { parentPort } = require('worker_threads');
const MarkdownTemplateEngine = require('./markdown-template-engine');
const engine = new MarkdownTemplateEngine();
parentPort.on('message', async ({ id, template, data, options }) => {
try {
const result = await engine.processTemplate(template, data);
parentPort.postMessage({ id, success: true, result });
} catch (error) {
parentPort.postMessage({ id, success: false, error: error.message });
}
});
`, { eval: true });
this.workers.push({
worker,
busy: false,
queue: []
});
}
}
async generateBatchOptimized(configurations, options = {}) {
const startTime = Date.now();
if (this.options.enableProfiling) {
console.log(`Starting batch generation of ${configurations.length} items...`);
}
// Build dependency graph
const dependencyGraph = this.buildDependencyGraph(configurations);
// Determine generation order
const generationOrder = this.topologicalSort(dependencyGraph);
// Process in batches with dependency awareness
const results = [];
const batches = this.createBatches(generationOrder, this.options.batchSize);
for (const batch of batches) {
const batchResults = await this.processBatch(batch, options);
results.push(...batchResults);
// Update cache and metrics
this.updateBatchMetrics(batch, batchResults);
}
const totalTime = Date.now() - startTime;
if (this.options.enableProfiling) {
console.log(`Batch generation completed in ${totalTime}ms`);
this.logPerformanceMetrics();
}
return {
results,
metrics: {
totalTime,
itemsGenerated: results.length,
itemsPerSecond: results.length / (totalTime / 1000),
cacheHits: this.metrics.get('cacheHits') || 0,
cacheMisses: this.metrics.get('cacheMisses') || 0
}
};
}
buildDependencyGraph(configurations) {
const graph = new Map();
configurations.forEach(config => {
const { id, dependencies = [], template, data } = config;
if (!graph.has(id)) {
graph.set(id, { config, dependencies: new Set(dependencies), dependents: new Set() });
}
// Build reverse dependencies
dependencies.forEach(depId => {
if (!graph.has(depId)) {
graph.set(depId, { config: null, dependencies: new Set(), dependents: new Set() });
}
graph.get(depId).dependents.add(id);
});
});
return graph;
}
topologicalSort(graph) {
const visited = new Set();
const visiting = new Set();
const result = [];
const visit = (nodeId) => {
if (visiting.has(nodeId)) {
throw new Error(`Circular dependency detected involving ${nodeId}`);
}
if (visited.has(nodeId)) {
return;
}
visiting.add(nodeId);
const node = graph.get(nodeId);
if (node && node.dependencies) {
node.dependencies.forEach(depId => visit(depId));
}
visiting.delete(nodeId);
visited.add(nodeId);
if (node && node.config) {
result.push(node.config);
}
};
graph.forEach((node, nodeId) => {
if (node.config) {
visit(nodeId);
}
});
return result;
}
createBatches(items, batchSize) {
const batches = [];
for (let i = 0; i < items.length; i += batchSize) {
batches.push(items.slice(i, i + batchSize));
}
return batches;
}
async processBatch(batch, options) {
const promises = batch.map(config => this.generateSingleOptimized(config, options));
const results = await Promise.allSettled(promises);
return results.map((result, index) => {
const config = batch[index];
if (result.status === 'fulfilled') {
return { ...config, ...result.value, success: true };
} else {
return {
...config,
success: false,
error: result.reason.message || result.reason
};
}
});
}
async generateSingleOptimized(config, options = {}) {
const { id, template, data, outputPath, cacheKey } = config;
const startTime = Date.now();
// Check cache first
if (cacheKey && this.options.incrementalGeneration) {
const cached = await this.checkCache(cacheKey);
if (cached) {
this.incrementMetric('cacheHits');
return { content: cached, fromCache: true, generationTime: 0 };
}
this.incrementMetric('cacheMisses');
}
try {
// Use worker pool for CPU-intensive template processing
const worker = await this.getAvailableWorker();
const result = await this.processWithWorker(worker, template, data);
const generationTime = Date.now() - startTime;
// Cache the result
if (cacheKey) {
await this.setCache(cacheKey, result);
}
// Write to file if output path specified
if (outputPath) {
await this.writeOutputFile(outputPath, result);
}
this.recordGenerationMetrics(id, generationTime);
return {
content: result,
generationTime,
fromCache: false
};
} catch (error) {
throw new Error(`Failed to generate ${id}: ${error.message}`);
}
}
async getAvailableWorker() {
const availableWorker = this.workers.find(w => !w.busy);
if (availableWorker) {
availableWorker.busy = true;
return availableWorker;
}
// Wait for a worker to become available
return new Promise((resolve) => {
const checkWorker = () => {
const worker = this.workers.find(w => !w.busy);
if (worker) {
worker.busy = true;
resolve(worker);
} else {
setTimeout(checkWorker, 10);
}
};
checkWorker();
});
}
async processWithWorker(workerInfo, template, data) {
const { worker } = workerInfo;
const taskId = Date.now().toString(36) + Math.random().toString(36).substr(2);
return new Promise((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error('Worker timeout'));
}, 30000);
const messageHandler = (message) => {
if (message.id === taskId) {
clearTimeout(timeout);
worker.off('message', messageHandler);
workerInfo.busy = false;
if (message.success) {
resolve(message.result);
} else {
reject(new Error(message.error));
}
}
};
worker.on('message', messageHandler);
worker.postMessage({ id: taskId, template, data });
});
}
async checkCache(key) {
if (this.cache instanceof Map) {
return this.cache.get(key);
} else {
return await this.cache.get(key);
}
}
async setCache(key, value) {
if (this.cache instanceof Map) {
this.cache.set(key, value);
} else {
await this.cache.set(key, value);
}
}
async writeOutputFile(outputPath, content) {
const fs = require('fs').promises;
const path = require('path');
await fs.mkdir(path.dirname(outputPath), { recursive: true });
if (this.options.compressionEnabled && outputPath.endsWith('.md')) {
const zlib = require('zlib');
const compressed = zlib.gzipSync(content);
await fs.writeFile(`${outputPath}.gz`, compressed);
} else {
await fs.writeFile(outputPath, content, 'utf-8');
}
}
incrementMetric(name) {
const current = this.metrics.get(name) || 0;
this.metrics.set(name, current + 1);
}
recordGenerationMetrics(id, generationTime) {
const times = this.metrics.get('generationTimes') || [];
times.push({ id, time: generationTime, timestamp: Date.now() });
this.metrics.set('generationTimes', times);
}
updateBatchMetrics(batch, results) {
const batchTime = Date.now();
const successful = results.filter(r => r.success).length;
const failed = results.length - successful;
this.incrementMetric('batchesProcessed');
this.incrementMetric('totalGenerated', successful);
this.incrementMetric('totalFailed', failed);
this.metrics.set('lastBatchTime', batchTime);
this.metrics.set('lastBatchSize', batch.length);
}
setupProfiling() {
this.profiler = {
marks: new Map(),
measures: new Map()
};
// Add performance marks
this.mark = (name) => {
this.profiler.marks.set(name, process.hrtime.bigint());
};
this.measure = (name, startMark) => {
const start = this.profiler.marks.get(startMark);
if (start) {
const duration = Number(process.hrtime.bigint() - start) / 1000000; // Convert to ms
this.profiler.measures.set(name, duration);
}
};
}
setupMetrics() {
// Initialize metric counters
const metricNames = [
'cacheHits', 'cacheMisses', 'batchesProcessed',
'totalGenerated', 'totalFailed', 'templateCompilations'
];
metricNames.forEach(name => this.metrics.set(name, 0));
this.metrics.set('startTime', Date.now());
this.metrics.set('generationTimes', []);
}
logPerformanceMetrics() {
const totalTime = Date.now() - this.metrics.get('startTime');
const totalGenerated = this.metrics.get('totalGenerated');
const cacheHitRate = this.metrics.get('cacheHits') /
(this.metrics.get('cacheHits') + this.metrics.get('cacheMisses')) * 100;
console.log('\n=== Performance Metrics ===');
console.log(`Total time: ${totalTime}ms`);
console.log(`Items generated: ${totalGenerated}`);
console.log(`Items per second: ${(totalGenerated / (totalTime / 1000)).toFixed(2)}`);
console.log(`Cache hit rate: ${cacheHitRate.toFixed(1)}%`);
console.log(`Batches processed: ${this.metrics.get('batchesProcessed')}`);
console.log(`Failed generations: ${this.metrics.get('totalFailed')}`);
// Generation time statistics
const times = this.metrics.get('generationTimes') || [];
if (times.length > 0) {
const timesOnly = times.map(t => t.time);
const avgTime = timesOnly.reduce((a, b) => a + b, 0) / timesOnly.length;
const maxTime = Math.max(...timesOnly);
const minTime = Math.min(...timesOnly);
console.log(`Avg generation time: ${avgTime.toFixed(2)}ms`);
console.log(`Max generation time: ${maxTime}ms`);
console.log(`Min generation time: ${minTime}ms`);
}
console.log('===========================\n');
}
getMetrics() {
return Object.fromEntries(this.metrics);
}
async cleanup() {
// Cleanup workers
await Promise.all(this.workers.map(w => w.worker.terminate()));
// Clear caches
if (this.cache instanceof Map) {
this.cache.clear();
} else if (this.cache.cleanup) {
await this.cache.cleanup();
}
}
}
// File-based cache implementation
class FileCache {
constructor(cacheDirectory) {
this.cacheDir = cacheDirectory;
this.fs = require('fs').promises;
this.path = require('path');
this.crypto = require('crypto');
}
async get(key) {
const filePath = this.getFilePath(key);
try {
const content = await this.fs.readFile(filePath, 'utf-8');
return JSON.parse(content);
} catch {
return null;
}
}
async set(key, value) {
const filePath = this.getFilePath(key);
await this.fs.mkdir(this.path.dirname(filePath), { recursive: true });
await this.fs.writeFile(filePath, JSON.stringify(value), 'utf-8');
}
getFilePath(key) {
const hash = this.crypto.createHash('md5').update(key).digest('hex');
return this.path.join(this.cacheDir, `${hash}.json`);
}
async cleanup() {
// Remove old cache files
try {
const files = await this.fs.readdir(this.cacheDir);
const cutoff = Date.now() - (7 * 24 * 60 * 60 * 1000); // 7 days
for (const file of files) {
const filePath = this.path.join(this.cacheDir, file);
const stats = await this.fs.stat(filePath);
if (stats.mtime.getTime() < cutoff) {
await this.fs.unlink(filePath);
}
}
} catch (error) {
console.warn('Cache cleanup failed:', error.message);
}
}
}
module.exports = PerformanceOptimizedGenerator;
Conclusion
Dynamic Markdown content generation represents a transformative approach to documentation management that enables organizations to build scalable, maintainable, and automatically synchronized documentation systems. By implementing sophisticated templating engines, data source integration capabilities, and performance-optimized generation pipelines, development teams can create documentation workflows that eliminate manual maintenance overhead while ensuring content accuracy and consistency across large-scale projects.
The key to successful dynamic content generation lies in designing flexible template architectures that can accommodate diverse content types and data sources while maintaining high performance and reliability. Whether you’re building API documentation systems, user guide generators, or comprehensive documentation portals, the techniques covered in this guide provide the foundation for creating robust, automated content generation systems that scale effectively with organizational growth.
Remember to implement proper caching strategies, dependency management, and performance monitoring to ensure that your dynamic generation systems remain responsive and reliable as content volume increases. With careful implementation of these advanced techniques, your documentation can achieve both the automation benefits of dynamic generation and the quality standards required for professional technical communication.