Bookmark this: tool.io/machine-learning/alphazero-puct/source-code
AlphaZero PUCT Demonstrator Source Code
Browse the implementation files for AlphaZero PUCT Demonstrator.tool.ts typescript
// Tool: AlphaZero PUCT Demonstrator
interface ChildStatistics {
move: string;
prior: number;
valueSum: number;
visits: number;
}
const MAX_CHILDREN = 1_000;
const parseFiniteNumber = (input: unknown): number | null => {
if (
(typeof input !== 'string' && typeof input !== 'number') ||
(typeof input === 'string' && input.trim() === '')
) {
return null;
}
const value = Number(input);
return Number.isFinite(value) ? value : null;
};
const parseNonNegativeInteger = (
input: unknown
): number | null => {
const value = parseFiniteNumber(input);
return value !== null &&
Number.isSafeInteger(value) &&
value >= 0
? value
: null;
};
const parseChildren = (
input: unknown
): ChildStatistics[] | null => {
if (typeof input !== 'string' || input.trim() === '')
return null;
try {
const children: unknown = JSON.parse(input);
if (
!Array.isArray(children) ||
children.length === 0 ||
children.length > MAX_CHILDREN
) {
return null;
}
const moves = new Set<string>();
for (const child of children) {
if (
typeof child !== 'object' ||
child === null ||
typeof child.move !== 'string' ||
child.move.trim() === '' ||
moves.has(child.move) ||
typeof child.prior !== 'number' ||
!Number.isFinite(child.prior) ||
child.prior < 0 ||
typeof child.visits !== 'number' ||
!Number.isSafeInteger(child.visits) ||
child.visits < 0 ||
typeof child.valueSum !== 'number' ||
!Number.isFinite(child.valueSum)
) {
return null;
}
moves.add(child.move);
}
return children as ChildStatistics[];
} catch {
return null;
}
};
const round = (value: number): number =>
Number(value.toPrecision(12));
export const demonstrateAlphaZeroPuct = (
childrenInput: unknown,
parentVisitsInput: unknown,
explorationInput: unknown,
temperatureInput: unknown
): string => {
const children = parseChildren(childrenInput);
const parentVisits = parseNonNegativeInteger(
parentVisitsInput
);
const exploration = parseFiniteNumber(explorationInput);
const temperature = parseFiniteNumber(temperatureInput);
if (
children === null ||
parentVisits === null ||
exploration === null ||
exploration <= 0 ||
temperature === null ||
temperature < 0
) {
return '';
}
const priorTotal = children.reduce(
(total, child) => total + child.prior,
0
);
if (!Number.isFinite(priorTotal) || priorTotal <= 0)
return '';
const scores = children.map((child) => {
const prior = child.prior / priorTotal;
const q =
child.visits === 0 ? 0 : child.valueSum / child.visits;
const u =
exploration *
prior *
(Math.sqrt(Math.max(1, parentVisits)) /
(1 + child.visits));
const score = q + u;
if (![q, u, score].every(Number.isFinite)) {
return null;
}
return {
move: child.move,
prior: round(prior),
q: round(q),
score: round(score),
u: round(u),
visits: child.visits
};
});
if (scores.some((score) => score === null)) return '';
const finiteScores = scores.filter(
(score): score is NonNullable<typeof score> =>
score !== null
);
const selected = finiteScores.reduce((best, candidate) =>
candidate.score > best.score ? candidate : best
);
let policy: Array<{ move: string; probability: number }>;
if (temperature === 0) {
const maximumVisits = Math.max(
...children.map((child) => child.visits)
);
const selectedPolicyIndex = children.findIndex(
(child) => child.visits === maximumVisits
);
policy = children.map((child, index) => ({
move: child.move,
probability: index === selectedPolicyIndex ? 1 : 0
}));
} else {
const logWeights = children.map((child) =>
child.visits === 0
? -Infinity
: Math.log(child.visits) / temperature
);
const maximumLogWeight = Math.max(...logWeights);
const weights = Number.isFinite(maximumLogWeight)
? logWeights.map((weight) =>
Math.exp(weight - maximumLogWeight)
)
: children.map(() => 0);
const weightTotal = weights.reduce(
(total, weight) => total + weight,
0
);
policy = children.map((child, index) => ({
move: child.move,
probability: round(
weightTotal === 0
? 1 / children.length
: weights[index] / weightTotal
)
}));
}
return JSON.stringify(
{
policy,
scores: finiteScores,
selectedMove: selected.move,
temperature
},
null,
2
);
};
const DEFAULT_CHILDREN = JSON.stringify([
{ move: 'A', prior: 0.5, visits: 12, valueSum: 7.2 },
{ move: 'B', prior: 0.3, visits: 5, valueSum: 2.5 },
{ move: 'C', prior: 0.2, visits: 1, valueSum: 0.1 }
]);
export const Tool: TypeTool = {
name: 'AlphaZero PUCT Demonstrator',
slug: 'machine-learning/alphazero-puct',
intro:
'Calculate AlphaZero-style PUCT action scores and a visit-count policy target for one search node.',
tool: demonstrateAlphaZeroPuct,
i: [
{
example: DEFAULT_CHILDREN,
label: 'Child Statistics',
placeholder:
'JSON array with move, prior, visits, and valueSum',
render: 'textarea',
type: 'text',
value: DEFAULT_CHILDREN
},
{
example: '18',
label: 'Parent Visits',
placeholder: 'Non-negative parent visit count',
type: 'number',
value: 18
},
{
example: '1.5',
label: 'PUCT Constant',
placeholder: 'Positive exploration constant',
type: 'number',
value: 1.5
},
{
example: '1',
help: 'Zero selects the most visited move; positive values soften the visit-count distribution.',
label: 'Temperature',
placeholder: 'Non-negative policy temperature',
type: 'number',
value: 1
}
],
o: [
{
example:
'{ "selectedMove": "A", "scores": [], "policy": [] }',
label: 'PUCT Result',
placeholder:
'PUCT scores and policy target will appear here...',
render: 'textarea',
type: 'text'
}
],
cite: [
{
title:
'Mastering Chess and Shogi by Self-Play with a General Reinforcement Learning Algorithm',
url: 'https://arxiv.org/abs/1712.01815'
},
{
title: 'AlphaZero — Wikipedia',
url: 'https://en.wikipedia.org/wiki/AlphaZero'
}
]
};
export default Tool;
Description
Browse the implementation files for AlphaZero PUCT Demonstrator.