Skip to content

Main content

Scaled Dot-Product Attention Demonstrator Source Code

Browse the implementation files for Scaled Dot-Product Attention Demonstrator.
tool.ts typescript
// Tool: Scaled Dot-Product Attention Demonstrator

type Mask = 'causal' | 'none';

const MASKS: Mask[] = ['none', 'causal'];
const MAX_COLUMNS = 200;
const MAX_ROWS = 500;
const MAX_WORK = 20_000_000;

const parseMatrix = (input: unknown): number[][] | null => {
	if (typeof input !== 'string' || input.trim() === '')
		return null;

	try {
		const matrix: unknown = JSON.parse(input);

		if (
			!Array.isArray(matrix) ||
			matrix.length === 0 ||
			matrix.length > MAX_ROWS ||
			!Array.isArray(matrix[0]) ||
			matrix[0].length === 0 ||
			matrix[0].length > MAX_COLUMNS
		) {
			return null;
		}

		const columns = matrix[0].length;

		return matrix.every(
			(row) =>
				Array.isArray(row) &&
				row.length === columns &&
				row.every(
					(value) =>
						typeof value === 'number' && Number.isFinite(value)
				)
		)
			? (matrix as number[][])
			: null;
	} catch {
		return null;
	}
};

const round = (value: number): number =>
	Number(value.toPrecision(12));

export const demonstrateScaledDotProductAttention = (
	queriesInput: unknown,
	keysInput: unknown,
	valuesInput: unknown,
	maskInput: unknown
): string => {
	const queries = parseMatrix(queriesInput);
	const keys = parseMatrix(keysInput);
	const values = parseMatrix(valuesInput);

	if (
		queries === null ||
		keys === null ||
		values === null ||
		!MASKS.includes(maskInput as Mask) ||
		queries[0].length !== keys[0].length ||
		keys.length !== values.length ||
		queries.length *
			keys.length *
			(queries[0].length + values[0].length) >
			MAX_WORK
	) {
		return '';
	}

	const mask = maskInput as Mask;
	const scale = Math.sqrt(keys[0].length);
	const scores: Array<Array<number | 'masked'>> = [];
	const weights: number[][] = [];
	const output: number[][] = [];

	for (
		let queryIndex = 0;
		queryIndex < queries.length;
		queryIndex += 1
	) {
		const rowScores = keys.map((key, keyIndex) => {
			if (mask === 'causal' && keyIndex > queryIndex) {
				return -Infinity;
			}

			return (
				queries[queryIndex].reduce(
					(total, value, dimension) =>
						total + value * key[dimension],
					0
				) / scale
			);
		});
		const finiteScores = rowScores.filter(Number.isFinite);

		if (finiteScores.length === 0) return '';

		const maximum = Math.max(...finiteScores);
		const exponentials = rowScores.map((score) =>
			Number.isFinite(score) ? Math.exp(score - maximum) : 0
		);
		const total = exponentials.reduce(
			(sum, value) => sum + value,
			0
		);

		if (!Number.isFinite(total) || total <= 0) return '';

		const rowWeights = exponentials.map(
			(value) => value / total
		);
		const rowOutput = values[0].map((_, valueDimension) =>
			rowWeights.reduce(
				(sum, weight, keyIndex) =>
					sum + weight * values[keyIndex][valueDimension],
				0
			)
		);

		if (![...rowWeights, ...rowOutput].every(Number.isFinite)) {
			return '';
		}

		scores.push(
			rowScores.map((score) =>
				Number.isFinite(score) ? round(score) : 'masked'
			)
		);
		weights.push(rowWeights.map(round));
		output.push(rowOutput.map(round));
	}

	return JSON.stringify(
		{
			mask,
			output,
			scale: round(scale),
			scores,
			weights
		},
		null,
		2
	);
};

export const Tool: TypeTool = {
	name: 'Scaled Dot-Product Attention Demonstrator',
	slug: 'machine-learning/scaled-dot-product-attention',
	intro:
		'Calculate scaled query–key scores, softmax weights, and weighted values with optional causal masking.',
	tool: demonstrateScaledDotProductAttention,
	i: [
		{
			example: '[[1,0],[0,1]]',
			label: 'Queries',
			placeholder: 'JSON query matrix',
			render: 'textarea',
			type: 'text',
			value: '[[1,0],[0,1]]'
		},
		{
			example: '[[1,0],[0,1]]',
			label: 'Keys',
			placeholder: 'JSON key matrix with the query width',
			render: 'textarea',
			type: 'text',
			value: '[[1,0],[0,1]]'
		},
		{
			example: '[[10,0],[0,20]]',
			label: 'Values',
			placeholder: 'JSON value matrix with one row per key',
			render: 'textarea',
			type: 'text',
			value: '[[10,0],[0,20]]'
		},
		{
			example: 'none',
			label: 'Mask',
			options: [
				{ selected: true, text: 'No mask', value: 'none' },
				{ text: 'Causal mask', value: 'causal' }
			],
			placeholder: 'Select an attention mask',
			type: 'select',
			value: 'none'
		}
	],
	o: [
		{
			example: '{ "weights": [], "output": [] }',
			label: 'Attention Result',
			placeholder:
				'Attention weights and output will appear here...',
			render: 'textarea',
			type: 'text'
		}
	],
	cite: [
		{
			title: 'Attention Is All You Need',
			url: 'https://arxiv.org/abs/1706.03762'
		},
		{
			title: 'Attention — Wikipedia',
			url: 'https://en.wikipedia.org/wiki/Attention_(machine_learning)'
		}
	]
};

export default Tool;

Description

Browse the implementation files for Scaled Dot-Product Attention Demonstrator.

References

  1. Tool.io on GitHub
  2. Tool.io TypeScript tools library

Similar or alternative tools