Skip to content

Main content

A3C Rollout Update Calculator Source Code

Browse the implementation files for A3C Rollout Update Calculator.
tool.ts typescript
// Tool: A3C Rollout Update Calculator

const NUMBER =
	/^[+-]?(?:(?:\d+(?:\.\d*)?)|(?:\.\d+))(?:[eE][+-]?\d+)?$/;

const parseFiniteNumber = (input: unknown): number | null => {
	if (
		(typeof input !== 'string' && typeof input !== 'number') ||
		(typeof input === 'string' && input.trim() === '')
	) {
		return null;
	}

	const value = Number(input);

	return Number.isFinite(value) ? value : null;
};

const parseNumberList = (input: unknown): number[] | null => {
	if (typeof input !== 'string' || input.trim() === '') return null;

	const tokens = input
		.split(/[,\n]/)
		.map((token) => token.trim())
		.filter((token) => token !== '');

	if (
		tokens.length === 0 ||
		tokens.some((token) => !NUMBER.test(token))
	) {
		return null;
	}

	const values = tokens.map(Number);

	return values.every(Number.isFinite) ? values : null;
};

const round = (value: number): number =>
	Number(value.toPrecision(12));

export const calculateA3cRolloutUpdate = (
	rewardsInput: unknown,
	valuesInput: unknown,
	actionLogProbabilitiesInput: unknown,
	entropiesInput: unknown,
	bootstrapValueInput: unknown,
	discountFactorInput: unknown,
	valueLossCoefficientInput: unknown,
	entropyCoefficientInput: unknown
): string => {
	const rewards = parseNumberList(rewardsInput);
	const values = parseNumberList(valuesInput);
	const actionLogProbabilities = parseNumberList(
		actionLogProbabilitiesInput
	);
	const entropies = parseNumberList(entropiesInput);
	const bootstrapValue = parseFiniteNumber(bootstrapValueInput);
	const discountFactor = parseFiniteNumber(discountFactorInput);
	const valueLossCoefficient = parseFiniteNumber(
		valueLossCoefficientInput
	);
	const entropyCoefficient = parseFiniteNumber(
		entropyCoefficientInput
	);

	if (
		rewards === null ||
		values === null ||
		actionLogProbabilities === null ||
		entropies === null ||
		bootstrapValue === null ||
		discountFactor === null ||
		valueLossCoefficient === null ||
		entropyCoefficient === null ||
		values.length !== rewards.length ||
		actionLogProbabilities.length !== rewards.length ||
		entropies.length !== rewards.length ||
		discountFactor < 0 ||
		discountFactor > 1 ||
		valueLossCoefficient < 0 ||
		entropyCoefficient < 0
	) {
		return '';
	}

	const returns = new Array<number>(rewards.length);
	let discountedReturn = bootstrapValue;

	for (let index = rewards.length - 1; index >= 0; index -= 1) {
		discountedReturn =
			rewards[index] + discountFactor * discountedReturn;
		if (!Number.isFinite(discountedReturn)) return '';
		returns[index] = discountedReturn;
	}

	const advantages = returns.map(
		(value, index) => value - values[index]
	);
	const actorLoss =
		-advantages.reduce(
			(total, advantage, index) =>
				total +
				actionLogProbabilities[index] * advantage +
				entropyCoefficient * entropies[index],
			0
		) / rewards.length;
	const criticLoss =
		advantages.reduce(
			(total, advantage) => total + advantage ** 2,
			0
		) /
		(2 * rewards.length);
	const totalLoss =
		actorLoss + valueLossCoefficient * criticLoss;

	if (
		![actorLoss, criticLoss, totalLoss, ...advantages].every(
			Number.isFinite
		)
	) {
		return '';
	}

	return JSON.stringify(
		{
			actorLoss: round(actorLoss),
			advantages: advantages.map(round),
			criticLoss: round(criticLoss),
			returns: returns.map(round),
			totalLoss: round(totalLoss)
		},
		null,
		2
	);
};

export const Tool: TypeTool = {
	name: 'A3C Rollout Update Calculator',
	slug: 'reinforcement-learning/a3c-rollout-update',
	intro:
		'Calculate discounted returns, advantages, and A3C rollout losses.',
	tool: calculateA3cRolloutUpdate,
	i: [
		{
			example: '1, 0, 2',
			label: 'Rewards',
			placeholder: 'Comma- or line-separated rewards',
			render: 'textarea',
			type: 'text',
			value: '1, 0, 2'
		},
		{
			example: '0.5, 0.4, 0.8',
			label: 'State Values',
			placeholder: 'One value estimate per reward',
			render: 'textarea',
			type: 'text',
			value: '0.5, 0.4, 0.8'
		},
		{
			example: '-0.2, -0.5, -0.1',
			label: 'Action Log Probabilities',
			placeholder: 'One action log probability per reward',
			render: 'textarea',
			type: 'text',
			value: '-0.2, -0.5, -0.1'
		},
		{
			example: '0.5, 0.6, 0.4',
			label: 'Policy Entropies',
			placeholder: 'One policy entropy per reward',
			render: 'textarea',
			type: 'text',
			value: '0.5, 0.6, 0.4'
		},
		{
			example: '0',
			label: 'Bootstrap Value',
			placeholder: 'Value after the rollout',
			type: 'number',
			value: 0
		},
		{
			example: '0.99',
			label: 'Discount Factor',
			placeholder: 'Gamma from 0 to 1',
			type: 'number',
			value: 0.99
		},
		{
			example: '0.5',
			label: 'Value Loss Coefficient',
			placeholder: 'Non-negative coefficient',
			type: 'number',
			value: 0.5
		},
		{
			example: '0.01',
			label: 'Entropy Coefficient',
			placeholder: 'Non-negative coefficient',
			type: 'number',
			value: 0.01
		}
	],
	o: [
		{
			example: '{ "returns": [], "advantages": [] }',
			label: 'A3C Rollout Update',
			placeholder: 'Calculated JSON will appear here...',
			render: 'textarea',
			type: 'text'
		}
	],
	cite: [
		{
			title:
				'Asynchronous Methods for Deep Reinforcement Learning',
			url: 'https://arxiv.org/abs/1602.01783'
		}
	]
};

export default Tool;

Description

Browse the implementation files for A3C Rollout Update Calculator.

References

  1. Tool.io on GitHub
  2. Tool.io TypeScript tools library

Similar or alternative tools