Scaled Dot-Product Attention Demonstrator Source Code
Browse the implementation files for Scaled Dot-Product Attention Demonstrator.tool.ts typescript
// Tool: Scaled Dot-Product Attention Demonstrator
type Mask = 'causal' | 'none';
const MASKS: Mask[] = ['none', 'causal'];
const MAX_COLUMNS = 200;
const MAX_ROWS = 500;
const MAX_WORK = 20_000_000;
const parseMatrix = (input: unknown): number[][] | null => {
if (typeof input !== 'string' || input.trim() === '')
return null;
try {
const matrix: unknown = JSON.parse(input);
if (
!Array.isArray(matrix) ||
matrix.length === 0 ||
matrix.length > MAX_ROWS ||
!Array.isArray(matrix[0]) ||
matrix[0].length === 0 ||
matrix[0].length > MAX_COLUMNS
) {
return null;
}
const columns = matrix[0].length;
return matrix.every(
(row) =>
Array.isArray(row) &&
row.length === columns &&
row.every(
(value) =>
typeof value === 'number' && Number.isFinite(value)
)
)
? (matrix as number[][])
: null;
} catch {
return null;
}
};
const round = (value: number): number =>
Number(value.toPrecision(12));
export const demonstrateScaledDotProductAttention = (
queriesInput: unknown,
keysInput: unknown,
valuesInput: unknown,
maskInput: unknown
): string => {
const queries = parseMatrix(queriesInput);
const keys = parseMatrix(keysInput);
const values = parseMatrix(valuesInput);
if (
queries === null ||
keys === null ||
values === null ||
!MASKS.includes(maskInput as Mask) ||
queries[0].length !== keys[0].length ||
keys.length !== values.length ||
queries.length *
keys.length *
(queries[0].length + values[0].length) >
MAX_WORK
) {
return '';
}
const mask = maskInput as Mask;
const scale = Math.sqrt(keys[0].length);
const scores: Array<Array<number | 'masked'>> = [];
const weights: number[][] = [];
const output: number[][] = [];
for (
let queryIndex = 0;
queryIndex < queries.length;
queryIndex += 1
) {
const rowScores = keys.map((key, keyIndex) => {
if (mask === 'causal' && keyIndex > queryIndex) {
return -Infinity;
}
return (
queries[queryIndex].reduce(
(total, value, dimension) =>
total + value * key[dimension],
0
) / scale
);
});
const finiteScores = rowScores.filter(Number.isFinite);
if (finiteScores.length === 0) return '';
const maximum = Math.max(...finiteScores);
const exponentials = rowScores.map((score) =>
Number.isFinite(score) ? Math.exp(score - maximum) : 0
);
const total = exponentials.reduce(
(sum, value) => sum + value,
0
);
if (!Number.isFinite(total) || total <= 0) return '';
const rowWeights = exponentials.map(
(value) => value / total
);
const rowOutput = values[0].map((_, valueDimension) =>
rowWeights.reduce(
(sum, weight, keyIndex) =>
sum + weight * values[keyIndex][valueDimension],
0
)
);
if (![...rowWeights, ...rowOutput].every(Number.isFinite)) {
return '';
}
scores.push(
rowScores.map((score) =>
Number.isFinite(score) ? round(score) : 'masked'
)
);
weights.push(rowWeights.map(round));
output.push(rowOutput.map(round));
}
return JSON.stringify(
{
mask,
output,
scale: round(scale),
scores,
weights
},
null,
2
);
};
export const Tool: TypeTool = {
name: 'Scaled Dot-Product Attention Demonstrator',
slug: 'machine-learning/scaled-dot-product-attention',
intro:
'Calculate scaled query–key scores, softmax weights, and weighted values with optional causal masking.',
tool: demonstrateScaledDotProductAttention,
i: [
{
example: '[[1,0],[0,1]]',
label: 'Queries',
placeholder: 'JSON query matrix',
render: 'textarea',
type: 'text',
value: '[[1,0],[0,1]]'
},
{
example: '[[1,0],[0,1]]',
label: 'Keys',
placeholder: 'JSON key matrix with the query width',
render: 'textarea',
type: 'text',
value: '[[1,0],[0,1]]'
},
{
example: '[[10,0],[0,20]]',
label: 'Values',
placeholder: 'JSON value matrix with one row per key',
render: 'textarea',
type: 'text',
value: '[[10,0],[0,20]]'
},
{
example: 'none',
label: 'Mask',
options: [
{ selected: true, text: 'No mask', value: 'none' },
{ text: 'Causal mask', value: 'causal' }
],
placeholder: 'Select an attention mask',
type: 'select',
value: 'none'
}
],
o: [
{
example: '{ "weights": [], "output": [] }',
label: 'Attention Result',
placeholder:
'Attention weights and output will appear here...',
render: 'textarea',
type: 'text'
}
],
cite: [
{
title: 'Attention Is All You Need',
url: 'https://arxiv.org/abs/1706.03762'
},
{
title: 'Attention — Wikipedia',
url: 'https://en.wikipedia.org/wiki/Attention_(machine_learning)'
}
]
};
export default Tool;
Description
Browse the implementation files for Scaled Dot-Product Attention Demonstrator.