Skip to content

Main content

AlphaZero PUCT Demonstrator Source Code

Browse the implementation files for AlphaZero PUCT Demonstrator.
tool.ts typescript
// Tool: AlphaZero PUCT Demonstrator

interface ChildStatistics {
	move: string;
	prior: number;
	valueSum: number;
	visits: number;
}

const MAX_CHILDREN = 1_000;

const parseFiniteNumber = (input: unknown): number | null => {
	if (
		(typeof input !== 'string' && typeof input !== 'number') ||
		(typeof input === 'string' && input.trim() === '')
	) {
		return null;
	}

	const value = Number(input);

	return Number.isFinite(value) ? value : null;
};

const parseNonNegativeInteger = (
	input: unknown
): number | null => {
	const value = parseFiniteNumber(input);

	return value !== null &&
		Number.isSafeInteger(value) &&
		value >= 0
		? value
		: null;
};

const parseChildren = (
	input: unknown
): ChildStatistics[] | null => {
	if (typeof input !== 'string' || input.trim() === '')
		return null;

	try {
		const children: unknown = JSON.parse(input);

		if (
			!Array.isArray(children) ||
			children.length === 0 ||
			children.length > MAX_CHILDREN
		) {
			return null;
		}

		const moves = new Set<string>();

		for (const child of children) {
			if (
				typeof child !== 'object' ||
				child === null ||
				typeof child.move !== 'string' ||
				child.move.trim() === '' ||
				moves.has(child.move) ||
				typeof child.prior !== 'number' ||
				!Number.isFinite(child.prior) ||
				child.prior < 0 ||
				typeof child.visits !== 'number' ||
				!Number.isSafeInteger(child.visits) ||
				child.visits < 0 ||
				typeof child.valueSum !== 'number' ||
				!Number.isFinite(child.valueSum)
			) {
				return null;
			}

			moves.add(child.move);
		}

		return children as ChildStatistics[];
	} catch {
		return null;
	}
};

const round = (value: number): number =>
	Number(value.toPrecision(12));

export const demonstrateAlphaZeroPuct = (
	childrenInput: unknown,
	parentVisitsInput: unknown,
	explorationInput: unknown,
	temperatureInput: unknown
): string => {
	const children = parseChildren(childrenInput);
	const parentVisits = parseNonNegativeInteger(
		parentVisitsInput
	);
	const exploration = parseFiniteNumber(explorationInput);
	const temperature = parseFiniteNumber(temperatureInput);

	if (
		children === null ||
		parentVisits === null ||
		exploration === null ||
		exploration <= 0 ||
		temperature === null ||
		temperature < 0
	) {
		return '';
	}

	const priorTotal = children.reduce(
		(total, child) => total + child.prior,
		0
	);

	if (!Number.isFinite(priorTotal) || priorTotal <= 0)
		return '';

	const scores = children.map((child) => {
		const prior = child.prior / priorTotal;
		const q =
			child.visits === 0 ? 0 : child.valueSum / child.visits;
		const u =
			exploration *
			prior *
			(Math.sqrt(Math.max(1, parentVisits)) /
				(1 + child.visits));

		const score = q + u;

		if (![q, u, score].every(Number.isFinite)) {
			return null;
		}

		return {
			move: child.move,
			prior: round(prior),
			q: round(q),
			score: round(score),
			u: round(u),
			visits: child.visits
		};
	});
	if (scores.some((score) => score === null)) return '';

	const finiteScores = scores.filter(
		(score): score is NonNullable<typeof score> =>
			score !== null
	);
	const selected = finiteScores.reduce((best, candidate) =>
		candidate.score > best.score ? candidate : best
	);
	let policy: Array<{ move: string; probability: number }>;

	if (temperature === 0) {
		const maximumVisits = Math.max(
			...children.map((child) => child.visits)
		);
		const selectedPolicyIndex = children.findIndex(
			(child) => child.visits === maximumVisits
		);
		policy = children.map((child, index) => ({
			move: child.move,
			probability: index === selectedPolicyIndex ? 1 : 0
		}));
	} else {
		const logWeights = children.map((child) =>
			child.visits === 0
				? -Infinity
				: Math.log(child.visits) / temperature
		);
		const maximumLogWeight = Math.max(...logWeights);
		const weights = Number.isFinite(maximumLogWeight)
			? logWeights.map((weight) =>
					Math.exp(weight - maximumLogWeight)
				)
			: children.map(() => 0);
		const weightTotal = weights.reduce(
			(total, weight) => total + weight,
			0
		);

		policy = children.map((child, index) => ({
			move: child.move,
			probability: round(
				weightTotal === 0
					? 1 / children.length
					: weights[index] / weightTotal
			)
		}));
	}

	return JSON.stringify(
		{
			policy,
			scores: finiteScores,
			selectedMove: selected.move,
			temperature
		},
		null,
		2
	);
};

const DEFAULT_CHILDREN = JSON.stringify([
	{ move: 'A', prior: 0.5, visits: 12, valueSum: 7.2 },
	{ move: 'B', prior: 0.3, visits: 5, valueSum: 2.5 },
	{ move: 'C', prior: 0.2, visits: 1, valueSum: 0.1 }
]);

export const Tool: TypeTool = {
	name: 'AlphaZero PUCT Demonstrator',
	slug: 'machine-learning/alphazero-puct',
	intro:
		'Calculate AlphaZero-style PUCT action scores and a visit-count policy target for one search node.',
	tool: demonstrateAlphaZeroPuct,
	i: [
		{
			example: DEFAULT_CHILDREN,
			label: 'Child Statistics',
			placeholder:
				'JSON array with move, prior, visits, and valueSum',
			render: 'textarea',
			type: 'text',
			value: DEFAULT_CHILDREN
		},
		{
			example: '18',
			label: 'Parent Visits',
			placeholder: 'Non-negative parent visit count',
			type: 'number',
			value: 18
		},
		{
			example: '1.5',
			label: 'PUCT Constant',
			placeholder: 'Positive exploration constant',
			type: 'number',
			value: 1.5
		},
		{
			example: '1',
			help: 'Zero selects the most visited move; positive values soften the visit-count distribution.',
			label: 'Temperature',
			placeholder: 'Non-negative policy temperature',
			type: 'number',
			value: 1
		}
	],
	o: [
		{
			example:
				'{ "selectedMove": "A", "scores": [], "policy": [] }',
			label: 'PUCT Result',
			placeholder:
				'PUCT scores and policy target will appear here...',
			render: 'textarea',
			type: 'text'
		}
	],
	cite: [
		{
			title:
				'Mastering Chess and Shogi by Self-Play with a General Reinforcement Learning Algorithm',
			url: 'https://arxiv.org/abs/1712.01815'
		},
		{
			title: 'AlphaZero — Wikipedia',
			url: 'https://en.wikipedia.org/wiki/AlphaZero'
		}
	]
};

export default Tool;

Description

Browse the implementation files for AlphaZero PUCT Demonstrator.

References

  1. Tool.io on GitHub
  2. Tool.io TypeScript tools library

Similar or alternative tools