A3C Rollout Update Calculator Source Code
Browse the implementation files for A3C Rollout Update Calculator.tool.ts typescript
// Tool: A3C Rollout Update Calculator
const NUMBER =
/^[+-]?(?:(?:\d+(?:\.\d*)?)|(?:\.\d+))(?:[eE][+-]?\d+)?$/;
const parseFiniteNumber = (input: unknown): number | null => {
if (
(typeof input !== 'string' && typeof input !== 'number') ||
(typeof input === 'string' && input.trim() === '')
) {
return null;
}
const value = Number(input);
return Number.isFinite(value) ? value : null;
};
const parseNumberList = (input: unknown): number[] | null => {
if (typeof input !== 'string' || input.trim() === '') return null;
const tokens = input
.split(/[,\n]/)
.map((token) => token.trim())
.filter((token) => token !== '');
if (
tokens.length === 0 ||
tokens.some((token) => !NUMBER.test(token))
) {
return null;
}
const values = tokens.map(Number);
return values.every(Number.isFinite) ? values : null;
};
const round = (value: number): number =>
Number(value.toPrecision(12));
export const calculateA3cRolloutUpdate = (
rewardsInput: unknown,
valuesInput: unknown,
actionLogProbabilitiesInput: unknown,
entropiesInput: unknown,
bootstrapValueInput: unknown,
discountFactorInput: unknown,
valueLossCoefficientInput: unknown,
entropyCoefficientInput: unknown
): string => {
const rewards = parseNumberList(rewardsInput);
const values = parseNumberList(valuesInput);
const actionLogProbabilities = parseNumberList(
actionLogProbabilitiesInput
);
const entropies = parseNumberList(entropiesInput);
const bootstrapValue = parseFiniteNumber(bootstrapValueInput);
const discountFactor = parseFiniteNumber(discountFactorInput);
const valueLossCoefficient = parseFiniteNumber(
valueLossCoefficientInput
);
const entropyCoefficient = parseFiniteNumber(
entropyCoefficientInput
);
if (
rewards === null ||
values === null ||
actionLogProbabilities === null ||
entropies === null ||
bootstrapValue === null ||
discountFactor === null ||
valueLossCoefficient === null ||
entropyCoefficient === null ||
values.length !== rewards.length ||
actionLogProbabilities.length !== rewards.length ||
entropies.length !== rewards.length ||
discountFactor < 0 ||
discountFactor > 1 ||
valueLossCoefficient < 0 ||
entropyCoefficient < 0
) {
return '';
}
const returns = new Array<number>(rewards.length);
let discountedReturn = bootstrapValue;
for (let index = rewards.length - 1; index >= 0; index -= 1) {
discountedReturn =
rewards[index] + discountFactor * discountedReturn;
if (!Number.isFinite(discountedReturn)) return '';
returns[index] = discountedReturn;
}
const advantages = returns.map(
(value, index) => value - values[index]
);
const actorLoss =
-advantages.reduce(
(total, advantage, index) =>
total +
actionLogProbabilities[index] * advantage +
entropyCoefficient * entropies[index],
0
) / rewards.length;
const criticLoss =
advantages.reduce(
(total, advantage) => total + advantage ** 2,
0
) /
(2 * rewards.length);
const totalLoss =
actorLoss + valueLossCoefficient * criticLoss;
if (
![actorLoss, criticLoss, totalLoss, ...advantages].every(
Number.isFinite
)
) {
return '';
}
return JSON.stringify(
{
actorLoss: round(actorLoss),
advantages: advantages.map(round),
criticLoss: round(criticLoss),
returns: returns.map(round),
totalLoss: round(totalLoss)
},
null,
2
);
};
export const Tool: TypeTool = {
name: 'A3C Rollout Update Calculator',
slug: 'reinforcement-learning/a3c-rollout-update',
intro:
'Calculate discounted returns, advantages, and A3C rollout losses.',
tool: calculateA3cRolloutUpdate,
i: [
{
example: '1, 0, 2',
label: 'Rewards',
placeholder: 'Comma- or line-separated rewards',
render: 'textarea',
type: 'text',
value: '1, 0, 2'
},
{
example: '0.5, 0.4, 0.8',
label: 'State Values',
placeholder: 'One value estimate per reward',
render: 'textarea',
type: 'text',
value: '0.5, 0.4, 0.8'
},
{
example: '-0.2, -0.5, -0.1',
label: 'Action Log Probabilities',
placeholder: 'One action log probability per reward',
render: 'textarea',
type: 'text',
value: '-0.2, -0.5, -0.1'
},
{
example: '0.5, 0.6, 0.4',
label: 'Policy Entropies',
placeholder: 'One policy entropy per reward',
render: 'textarea',
type: 'text',
value: '0.5, 0.6, 0.4'
},
{
example: '0',
label: 'Bootstrap Value',
placeholder: 'Value after the rollout',
type: 'number',
value: 0
},
{
example: '0.99',
label: 'Discount Factor',
placeholder: 'Gamma from 0 to 1',
type: 'number',
value: 0.99
},
{
example: '0.5',
label: 'Value Loss Coefficient',
placeholder: 'Non-negative coefficient',
type: 'number',
value: 0.5
},
{
example: '0.01',
label: 'Entropy Coefficient',
placeholder: 'Non-negative coefficient',
type: 'number',
value: 0.01
}
],
o: [
{
example: '{ "returns": [], "advantages": [] }',
label: 'A3C Rollout Update',
placeholder: 'Calculated JSON will appear here...',
render: 'textarea',
type: 'text'
}
],
cite: [
{
title:
'Asynchronous Methods for Deep Reinforcement Learning',
url: 'https://arxiv.org/abs/1602.01783'
}
]
};
export default Tool;
Description
Browse the implementation files for A3C Rollout Update Calculator.