Skip to content

Main content

Adam Optimizer Step Calculator Source Code

Browse the implementation files for Adam Optimizer Step Calculator.
tool.ts typescript
// Tool: Adam Optimizer Step Calculator

const parseFiniteNumber = (input: unknown): number | null => {
	if (
		(typeof input !== 'string' && typeof input !== 'number') ||
		(typeof input === 'string' && input.trim() === '')
	) {
		return null;
	}

	const value = Number(input);

	return Number.isFinite(value) ? value : null;
};

const parsePositiveInteger = (input: unknown): number | null => {
	const value = parseFiniteNumber(input);

	return value !== null && Number.isSafeInteger(value) && value >= 1
		? value
		: null;
};

const round = (value: number): number =>
	Number(value.toPrecision(12));

export const calculateAdamStep = (
	parameterInput: unknown,
	gradientInput: unknown,
	firstMomentInput: unknown,
	secondMomentInput: unknown,
	stepInput: unknown,
	learningRateInput: unknown,
	betaOneInput: unknown,
	betaTwoInput: unknown,
	epsilonInput: unknown
): string => {
	const parameter = parseFiniteNumber(parameterInput);
	const gradient = parseFiniteNumber(gradientInput);
	const firstMoment = parseFiniteNumber(firstMomentInput);
	const secondMoment = parseFiniteNumber(secondMomentInput);
	const step = parsePositiveInteger(stepInput);
	const learningRate = parseFiniteNumber(learningRateInput);
	const betaOne = parseFiniteNumber(betaOneInput);
	const betaTwo = parseFiniteNumber(betaTwoInput);
	const epsilon = parseFiniteNumber(epsilonInput);

	if (
		parameter === null ||
		gradient === null ||
		firstMoment === null ||
		secondMoment === null ||
		step === null ||
		learningRate === null ||
		betaOne === null ||
		betaTwo === null ||
		epsilon === null ||
		secondMoment < 0 ||
		learningRate <= 0 ||
		betaOne < 0 ||
		betaOne >= 1 ||
		betaTwo < 0 ||
		betaTwo >= 1 ||
		epsilon <= 0
	) {
		return '';
	}

	const nextFirstMoment =
		betaOne * firstMoment + (1 - betaOne) * gradient;
	const nextSecondMoment =
		betaTwo * secondMoment +
		(1 - betaTwo) * gradient ** 2;
	const correctedFirstMoment =
		nextFirstMoment / (1 - betaOne ** step);
	const correctedSecondMoment =
		nextSecondMoment / (1 - betaTwo ** step);
	const update =
		(learningRate * correctedFirstMoment) /
		(Math.sqrt(correctedSecondMoment) + epsilon);
	const nextParameter = parameter - update;

	if (
		![
			correctedFirstMoment,
			correctedSecondMoment,
			nextFirstMoment,
			nextParameter,
			nextSecondMoment,
			update
		].every(Number.isFinite)
	) {
		return '';
	}

	return JSON.stringify(
		{
			correctedFirstMoment: round(correctedFirstMoment),
			correctedSecondMoment: round(correctedSecondMoment),
			firstMoment: round(nextFirstMoment),
			parameter: round(nextParameter),
			secondMoment: round(nextSecondMoment),
			step,
			update: round(update)
		},
		null,
		2
	);
};

export const Tool: TypeTool = {
	name: 'Adam Optimizer Step Calculator',
	slug: 'optimization/adam-step',
	intro:
		'Calculate one scalar Adam parameter update with bias-corrected moments.',
	tool: calculateAdamStep,
	i: [
		{
			example: '1',
			label: 'Parameter',
			placeholder: 'Current parameter value',
			type: 'number',
			value: 1
		},
		{
			example: '0.1',
			label: 'Gradient',
			placeholder: 'Current gradient',
			type: 'number',
			value: 0.1
		},
		{
			example: '0',
			label: 'First Moment',
			placeholder: 'Previous first moment',
			type: 'number',
			value: 0
		},
		{
			example: '0',
			label: 'Second Moment',
			placeholder: 'Previous non-negative second moment',
			type: 'number',
			value: 0
		},
		{
			example: '1',
			label: 'Step',
			placeholder: 'Positive optimization step',
			type: 'number',
			value: 1
		},
		{
			example: '0.001',
			label: 'Learning Rate',
			placeholder: 'Positive learning rate',
			type: 'number',
			value: 0.001
		},
		{
			example: '0.9',
			label: 'Beta 1',
			placeholder: 'First-moment decay from 0 up to 1',
			type: 'number',
			value: 0.9
		},
		{
			example: '0.999',
			label: 'Beta 2',
			placeholder: 'Second-moment decay from 0 up to 1',
			type: 'number',
			value: 0.999
		},
		{
			example: '0.00000001',
			label: 'Epsilon',
			placeholder: 'Positive stability constant',
			type: 'number',
			value: 0.00000001
		}
	],
	o: [
		{
			example: '{ "parameter": 0.999, "update": 0.001 }',
			label: 'Adam Step',
			placeholder: 'Updated optimizer state will appear here...',
			render: 'textarea',
			type: 'text'
		}
	],
	cite: [
		{
			title: 'Adam: A Method for Stochastic Optimization',
			url: 'https://arxiv.org/abs/1412.6980'
		}
	]
};

export default Tool;

Description

Browse the implementation files for Adam Optimizer Step Calculator.

References

  1. Tool.io on GitHub
  2. Tool.io TypeScript tools library

Similar or alternative tools