Skip to content

Main content

AdaDelta Optimizer Step Calculator Source Code

Browse the implementation files for AdaDelta Optimizer Step Calculator.
tool.ts typescript
// Tool: AdaDelta Optimizer Step Calculator

const parseFiniteNumber = (input: unknown): number | null => {
	if (
		(typeof input !== 'string' && typeof input !== 'number') ||
		(typeof input === 'string' && input.trim() === '')
	) {
		return null;
	}

	const value = Number(input);

	return Number.isFinite(value) ? value : null;
};

const round = (value: number): number =>
	Number(value.toPrecision(12));

export const calculateAdaDeltaStep = (
	parameterInput: unknown,
	gradientInput: unknown,
	averageSquaredGradientInput: unknown,
	averageSquaredUpdateInput: unknown,
	decayRateInput: unknown,
	epsilonInput: unknown
): string => {
	const parameter = parseFiniteNumber(parameterInput);
	const gradient = parseFiniteNumber(gradientInput);
	const averageSquaredGradient = parseFiniteNumber(
		averageSquaredGradientInput
	);
	const averageSquaredUpdate = parseFiniteNumber(
		averageSquaredUpdateInput
	);
	const decayRate = parseFiniteNumber(decayRateInput);
	const epsilon = parseFiniteNumber(epsilonInput);

	if (
		parameter === null ||
		gradient === null ||
		averageSquaredGradient === null ||
		averageSquaredUpdate === null ||
		decayRate === null ||
		epsilon === null ||
		averageSquaredGradient < 0 ||
		averageSquaredUpdate < 0 ||
		decayRate < 0 ||
		decayRate >= 1 ||
		epsilon <= 0
	) {
		return '';
	}

	const nextAverageSquaredGradient =
		decayRate * averageSquaredGradient +
		(1 - decayRate) * gradient ** 2;
	const update =
		-Math.sqrt(
			(averageSquaredUpdate + epsilon) /
				(nextAverageSquaredGradient + epsilon)
		) * gradient;
	const nextAverageSquaredUpdate =
		decayRate * averageSquaredUpdate +
		(1 - decayRate) * update ** 2;
	const nextParameter = parameter + update;

	if (
		![
			nextAverageSquaredGradient,
			nextAverageSquaredUpdate,
			nextParameter,
			update
		].every(Number.isFinite)
	) {
		return '';
	}

	return JSON.stringify(
		{
			averageSquaredGradient: round(
				nextAverageSquaredGradient
			),
			averageSquaredUpdate: round(nextAverageSquaredUpdate),
			parameter: round(nextParameter),
			update: round(update)
		},
		null,
		2
	);
};

export const Tool: TypeTool = {
	name: 'AdaDelta Optimizer Step Calculator',
	slug: 'optimization/adadelta-step',
	intro:
		'Calculate one scalar AdaDelta parameter update and its accumulated state.',
	tool: calculateAdaDeltaStep,
	i: [
		{
			example: '1',
			label: 'Parameter',
			placeholder: 'Current parameter value',
			type: 'number',
			value: 1
		},
		{
			example: '0.1',
			label: 'Gradient',
			placeholder: 'Current gradient',
			type: 'number',
			value: 0.1
		},
		{
			example: '0',
			label: 'Average Squared Gradient',
			placeholder: 'Previous accumulated gradient state',
			type: 'number',
			value: 0
		},
		{
			example: '0',
			label: 'Average Squared Update',
			placeholder: 'Previous accumulated update state',
			type: 'number',
			value: 0
		},
		{
			example: '0.95',
			label: 'Decay Rate',
			placeholder: 'Rho from 0 up to 1',
			type: 'number',
			value: 0.95
		},
		{
			example: '0.000001',
			label: 'Epsilon',
			placeholder: 'Positive stability constant',
			type: 'number',
			value: 0.000001
		}
	],
	o: [
		{
			example: '{ "parameter": 0.9955, "update": -0.0045 }',
			label: 'AdaDelta Step',
			placeholder: 'Updated optimizer state will appear here...',
			render: 'textarea',
			type: 'text'
		}
	],
	cite: [
		{
			title: 'ADADELTA: An Adaptive Learning Rate Method',
			url: 'https://arxiv.org/abs/1212.5701'
		}
	]
};

export default Tool;

Description

Browse the implementation files for AdaDelta Optimizer Step Calculator.

References

  1. Tool.io on GitHub
  2. Tool.io TypeScript tools library

Similar or alternative tools