Better template variable extraction

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ztimson/utils",
-	"version": "0.27.15",
+	"version": "0.28.1",
 	"description": "Utility library",
 	"author": "Zak Timson",
 	"license": "MIT",

src/index.ts

@@ -19,6 +19,7 @@ export * from './path-events';
 export * from './promise-progress';
 export * from './search';
 export * from './string';
+export * from './template';
 export * from './time';
 export * from './types';
 export * from 'var-persist';

src/template.ts

@@ -0,0 +1,157 @@
+import {BadRequestError} from './errors.ts';
+import {dotNotation} from './objects.ts';
+import {matchAll} from './string.ts';
+import {formatDate} from './time.ts';
+
+export class TemplateError extends BadRequestError { }
+
+export function findTemplateVars(html: string): Record<string, any> {
+	const variables = new Set<string>();
+	const excluded = new Set<string>(['true', 'false', 'null', 'undefined']);
+
+	// Extract & exclude loop variables
+	for (const loop of matchAll(html, /\{\{\s*?\*\s*?(.+?)\s+in\s+(.+?)\s*?}}/g)) {
+		const [element, index = 'index'] = loop[1].replaceAll(/[()\s]/g, '').split(',');
+		excluded.add(element);
+		excluded.add(index);
+		// Add the array reference (but exclude if it's a loop variable)
+		const arrayRef = loop[2].trim();
+		const arrayVar = arrayRef.split('.')[0].match(/^[a-zA-Z_$][a-zA-Z0-9_$]*/)?.[0];
+		if (arrayVar && !excluded.has(arrayVar)) variables.add(arrayVar);
+	}
+
+	// Extract variables from if/else-if conditions
+	for (const ifStmt of matchAll(html, /\{\{\s*?[!]?\?\s*?([^}]+?)\s*?}}/g)) {
+		const code = ifStmt[1].replace(/["'`][^"'`]*["'`]/g, ''); // Remove string literals
+		const vars = code.match(/[a-zA-Z_$][a-zA-Z0-9_$.]+/g) || [];
+		for (const v of vars) {
+			const root = v.split('.')[0];
+			if (!excluded.has(root)) variables.add(root);
+		}
+	}
+
+	// Extract variables from content within if blocks
+	for (const ifBlock of matchAll(html, /\{\{\s*?\?\s*?.+?\s*?}}([\s\S]*?)\{\{\s*?\/\?\s*?}}/g)) {
+		const content = ifBlock[1];
+		const regex = /\{\{\s*([^<>\*\?!/}\s][^}]*?)\s*}}/g;
+		let match;
+		while ((match = regex.exec(content)) !== null) {
+			const code = match[1].trim().replace(/["'`][^"'`]*["'`]/g, '');
+			const vars = code.match(/[a-zA-Z_$][a-zA-Z0-9_$.]+/g) || [];
+			for (const v of vars) {
+				const root = v.split('.')[0];
+				if (!excluded.has(root)) variables.add(v);
+			}
+		}
+	}
+
+	// Extract variables from regular interpolations
+	const regex = /\{\{\s*([^<>\*\?!/}\s][^}]*?)\s*}}/g;
+	let match;
+	while ((match = regex.exec(html)) !== null) {
+		const code = match[1].trim().replace(/["'`][^"'`]*["'`]/g, '');
+		const vars = code.match(/[a-zA-Z_$][a-zA-Z0-9_$.]+/g) || [];
+		for (const v of vars) {
+			const root = v.split('.')[0];
+			if (!excluded.has(root)) variables.add(v);
+		}
+	}
+
+	const result: Record<string, any> = {};
+	for (const path of variables) {
+		const parts = path.split('.');
+		let current = result;
+		for (let i = 0; i < parts.length; i++) {
+			const part = parts[i];
+			if (i === parts.length - 1) {
+				current[part] = '';
+			} else {
+				current[part] = current[part] || {};
+				current = current[part];
+			}
+		}
+	}
+	return result;
+}
+
+export async function renderTemplate(template: string, data: any, fetch?: (file: string) => Promise<string>) {
+	let content = template, found: any;
+	const now = new Date(), d = {
+		date: {
+			day: now.getDate(),
+			month: now.toLocaleString('default', { month: 'long' }),
+			year: now.getFullYear(),
+			time: now.toLocaleTimeString(),
+			format: formatDate
+		},
+		...(data || {}),
+	};
+	if(!fetch) fetch = (file) => { throw new TemplateError(`Unable to fetch template: ${file}`); }
+	const evaluate = (code: string, data: object, fatal = true) => {
+		try {
+			return Function('data', `with(data) { return ${code}; }`)(data);
+		} catch {
+			if(fatal) throw new TemplateError(`Failed to evaluate: ${code}`);
+			else return false;
+		}
+	}
+
+	// If Statements - Optimize what we render: `{{ ? javascript }} IF TRUE CONTENT {{ !? javascript }} ELSE-IF CONTENT {{ !? }} ELSE FALSE CONTENT {{ /? }}`
+	while(!!(found = /\{\{\s*?\?\s*?(.+?)\s*?}}([\s\S]*?)\{\{\s*?\/\?\s*?}}/g.exec(content))) {
+		const nested = matchAll(found[0], /\{\{\s*?\?.+?}}/g).slice(-1)?.[0]?.index;
+		if(nested != 0) found = /\{\{\s*?\?\s*?(.+?)\s*?}}([\s\S]*?)\{\{\s*?\/\?\s*?}}/g.exec(content.slice(found.index + nested))
+		const parts = found[2].split(/\{\{\s*?!\?\s*?/);
+		let result = evaluate(found[1], d, false) ? parts[0] : '';
+		if (!result) {
+			for (let i = 1; i < parts.length; i++) {
+				const [cond, body] = parts[i].split(/}}/);
+				if (!cond.trim()) {
+					result = body || '';
+					break;
+				}
+				if (evaluate(cond, d, false)) {
+					result = body || '';
+					break;
+				}
+			}
+		}
+		content = content.replace(found[0], result);
+	}
+
+	// Imports - We render bottom up: `{{ < file.html }}`
+	while(!!(found = /\{\{\s*?<\s*?(.+?)\s*?}}/g.exec(content))) {
+		const t = await fetch(found[1].trim());
+		if(!t) throw new TemplateError(`Unknown imported template: ${found[1].trim()}`);
+		content = content.replace(found[0], await renderTemplate(t, d, fetch));
+	}
+
+	// For Loops: `{{ * (row, index) in invoice }} CONTENT {{ /* }}`
+	while(!!(found = /\{\{\s*?\*\s*?(.+?)\s+in\s+(.+?)\s*?}}([\s\S]*?)\{\{\s*?\/\*\s*?}}/g.exec(content))) {
+		const split = found[1].replaceAll(/[()\s]/g, '').split(',');
+		const element = split[0];
+		const index = split[1] || 'index';
+		const array: any[] = <any>dotNotation(d, found[2]);
+		if(!array || typeof array != 'object') throw new TemplateError(`Cannot iterate: ${found[2]}`);
+		let compiled = [];
+		for(let i = 0; i < array.length; i++)
+			compiled.push(await renderTemplate(found[3], {...d, [element]: array[i], [index]: i}, fetch));
+		content = content.replace(found[0], compiled.join('\n'));
+	}
+
+	// Evaluate whatever is left - Should come last: `{{ javascript }}`
+	while(!!(found = /\{\{\s*([^<>\*\?!/}\s][^}]*?)\s*}}/g.exec(content))) {
+		content = content.replace(found[0], evaluate(found[1].trim(), d) ?? '');
+	}
+
+	// Extends: `{{ > file.html:property }} CONTENT {{ /> }}`
+	while(!!(found = /\{\{\s*?>\s*?(.+?):(.+?)\s*?}}([\s\S]*?)\{\{\s*?\/>\s*?}}/g.exec(content))) {
+		const t = await fetch(found[1].trim());
+		if(!t) throw new TemplateError(`Unknown extended templated: ${found[1].trim()}`);
+		content = content.replace(found[0], await renderTemplate(t, {
+			...d,
+			[found[2].trim()]: found[3],
+		}, fetch));
+	}
+
+	return content;
+}

tests/templates.spec.ts

@@ -0,0 +1,215 @@
+import {findTemplateVars, renderTemplate, TemplateError} from '../src';
+
+describe('findTemplateVars', () => {
+	test('extracts simple variables', () => {
+		const result = findTemplateVars('Hello {{ name }}!');
+		expect(result).toEqual({ name: '' });
+	});
+
+	test('extracts nested object paths', () => {
+		const result = findTemplateVars('{{ user.name }} is {{ user.age }}');
+		expect(result).toEqual({ user: { name: '', age: '' } });
+	});
+
+	test('extracts variables from if statements', () => {
+		const result = findTemplateVars('{{ ? active }}{{ message }}{{ /? }}');
+		expect(result).toEqual({ active: '', message: '' });
+	});
+
+	test('extracts variables from else-if conditions', () => {
+		const result = findTemplateVars('{{ ? status == "paid" }}PAID{{ !? status == "pending" }}{{ value }}{{ /? }}');
+		expect(result).toEqual({ status: '', value: '' });
+	});
+
+	test('extracts array reference from loops', () => {
+		const result = findTemplateVars('{{ * item in items }}{{ item }}{{ /* }}');
+		expect(result).toEqual({ items: '' });
+	});
+
+	test('excludes loop element variable', () => {
+		const result = findTemplateVars('{{ * item in items }}{{ item.name }}{{ /* }}');
+		expect(result).toEqual({ items: '' });
+		expect(result).not.toHaveProperty('item');
+	});
+
+	test('excludes loop index variable', () => {
+		const result = findTemplateVars('{{ * (item, i) in items }}{{ i }}:{{ item }}{{ /* }}');
+		expect(result).toEqual({ items: '' });
+		expect(result).not.toHaveProperty('item');
+		expect(result).not.toHaveProperty('i');
+	});
+
+	test('extracts external vars used inside loops', () => {
+		const result = findTemplateVars('{{ * item in items }}{{ item }}-{{ prefix }}{{ /* }}');
+		expect(result).toEqual({ items: '', prefix: '' });
+	});
+
+	test('handles nested loops', () => {
+		const result = findTemplateVars('{{ * row in rows }}{{ * col in row.cols }}{{ col }}{{ /* }}{{ /* }}');
+		expect(result).toEqual({ rows: '' });
+		expect(result).not.toHaveProperty('row');
+		expect(result).not.toHaveProperty('col');
+	});
+
+	test('extracts from complex nested template', () => {
+		const tpl = `
+{{ ? items.length > 0 }}
+{{ * (item, i) in items }}
+{{ i }}. {{ item.name }}: {{ currency }}{{ item.price }}
+{{ /* }}
+Total: {{ total }}
+{{ !? }}
+{{ emptyMessage }}
+{{ /? }}`;
+		const result = findTemplateVars(tpl);
+		expect(result).toEqual({
+			items: '',
+			currency: '',
+			total: '',
+			emptyMessage: ''
+		});
+		expect(result).not.toHaveProperty('item');
+		expect(result).not.toHaveProperty('i');
+	});
+
+	test('ignores template control syntax', () => {
+		const result = findTemplateVars('{{ < header.html }}{{ > layout.html:content }}content{{ /> }}');
+		expect(result).toEqual({});
+	});
+
+	test('handles multiple variables in expressions', () => {
+		const result = findTemplateVars('{{ firstName + " " + lastName }}');
+		expect(result).toEqual({ firstName: '', lastName: '' });
+	});
+});
+
+describe('renderTemplate', () => {
+	test('basic variable interpolation', async () => {
+		const result = await renderTemplate('Hello {{ name }}!', { name: 'World' });
+		expect(result).toBe('Hello World!');
+	});
+
+	test('nested object access', async () => {
+		const result = await renderTemplate('{{ user.name }} is {{ user.age }}', {
+			user: { name: 'Alice', age: 25 }
+		});
+		expect(result).toBe('Alice is 25');
+	});
+
+	test('method calls', async () => {
+		const result = await renderTemplate('{{ price.toFixed(2) }}', { price: 9.5 });
+		expect(result).toBe('9.50');
+	});
+
+	test('if statement true', async () => {
+		const result = await renderTemplate('{{ ? active }}YES{{ /? }}', { active: true });
+		expect(result).toBe('YES');
+	});
+
+	test('if statement false', async () => {
+		const result = await renderTemplate('{{ ? active }}YES{{ /? }}', { active: false });
+		expect(result).toBe('');
+	});
+
+	test('if-else', async () => {
+		const result = await renderTemplate('{{ ? active }}YES{{ !? }}NO{{ /? }}', { active: false });
+		expect(result).toBe('NO');
+	});
+
+	test('if-elseif-else', async () => {
+		const tpl = '{{ ? status == "paid" }}PAID{{ !? status == "pending" }}PENDING{{ !? }}OTHER{{ /? }}';
+		expect(await renderTemplate(tpl, { status: 'paid' })).toBe('PAID');
+		expect(await renderTemplate(tpl, { status: 'pending' })).toBe('PENDING');
+		expect(await renderTemplate(tpl, { status: 'failed' })).toBe('OTHER');
+	});
+
+	test('nested if statements', async () => {
+		const tpl = '{{ ? a }}A{{ ? b }}B{{ /? }}{{ /? }}';
+		expect(await renderTemplate(tpl, { a: true, b: true })).toBe('AB');
+		expect(await renderTemplate(tpl, { a: true, b: false })).toBe('A');
+		expect(await renderTemplate(tpl, { a: false, b: true })).toBe('');
+	});
+
+	test('for loop', async () => {
+		const tpl = '{{ * item in items }}{{ item }}{{ /* }}';
+		const result = await renderTemplate(tpl, { items: ['a', 'b', 'c'] });
+		expect(result).toBe('a\nb\nc');
+	});
+
+	test('for loop with index', async () => {
+		const tpl = '{{ * (item, i) in items }}{{ i }}:{{ item }}{{ /* }}';
+		const result = await renderTemplate(tpl, { items: ['a', 'b'] });
+		expect(result).toBe('0:a\n1:b');
+	});
+
+	test('for loop with objects', async () => {
+		const tpl = '{{ * user in users }}{{ user.name }}{{ /* }}';
+		const result = await renderTemplate(tpl, {
+			users: [{ name: 'Alice' }, { name: 'Bob' }]
+		});
+		expect(result).toBe('Alice\nBob');
+	});
+
+	test('for loop error on non-array', async () => {
+		await expect(
+			renderTemplate('{{ * x in notArray }}{{ x }}{{ /* }}', { notArray: 'string' })
+		).rejects.toThrow(TemplateError);
+	});
+
+	test('import template', async () => {
+		const fetch = async (file: string) => {
+			if (file === 'header.html') return 'HEADER: {{ title }}';
+			throw new Error('Not found');
+		};
+		const result = await renderTemplate('{{ < header.html }}', { title: 'Test' }, fetch);
+		expect(result).toBe('HEADER: Test');
+	});
+
+	test('import template error', async () => {
+		await expect(
+			renderTemplate('{{ < missing.html }}', {})
+		).rejects.toThrow(TemplateError);
+	});
+
+	test('extend template', async () => {
+		const fetch = async (file: string) => {
+			if (file === 'layout.html') return '<div>{{ content }}</div>';
+			throw new Error('Not found');
+		};
+		const result = await renderTemplate('{{ > layout.html:content }}Hello{{ /> }}', {}, fetch);
+		expect(result).toBe('<div>Hello</div>');
+	});
+
+	test('date object', async () => {
+		const result = await renderTemplate('{{ date.year }}', {});
+		expect(result).toBe(new Date().getFullYear().toString());
+	});
+
+	test('evaluation error', async () => {
+		await expect(
+			renderTemplate('{{ undefined.property }}', {})
+		).rejects.toThrow(TemplateError);
+	});
+
+	test('complex nested example', async () => {
+		const tpl = `
+{{ ? items.length > 0 }}
+{{ * (item, i) in items }}
+{{ i + 1 }}. {{ item.name }}: \${{ item.price.toFixed(2) }}
+{{ /* }}
+Total: \${{ total.toFixed(2) }}
+{{ !? }}
+No items
+{{ /? }}`;
+		const result = await renderTemplate(tpl, {
+			items: [
+				{ name: 'A', price: 10 },
+				{ name: 'B', price: 20 }
+			],
+			total: 30
+		});
+		expect(result.trim()).toContain('1. A: $10.00');
+		expect(result.trim()).toContain('2. B: $20.00');
+		expect(result.trim()).toContain('Total: $30.00');
+	});
+});