feat: inject controllers using metadata's definition of pallets (#1592)

* adds naming to AbstractController and to controllers

* wip:injected controllers

* inject controllers using metadata

* inject controllers using metadata

* removed unused method

* injects controllers using pallets in metadata

* lint

* update requiredPallets abstract definition

* update injected flag to make injected controllers not default (no breaking changes)

* Update README.md

Co-authored-by: Dominique <dominique@imodworks.io>

* Update src/chains-config/index.ts

Co-authored-by: Dominique <dominique@imodworks.io>

* fix default options

* remove log

* update readme

* Update src/controllers/coretime/CoretimeChainController.ts

Co-authored-by: Dominique <dominique@imodworks.io>

* Update src/controllers/coretime/CoretimeGenericController.ts

Co-authored-by: Dominique <dominique@imodworks.io>

* linting

* rebase and fix tests

---------

Co-authored-by: Dominique <dominique@imodworks.io>

README.md

@@ -165,6 +165,7 @@ For more information on our configuration manager visit its readme [here](https:
 - `SAS_EXPRESS_PORT`: port on which the server will be listening, defaults to `8080`.
 - `SAS_EXPRESS_KEEP_ALIVE_TIMEOUT`: Set the `keepAliveTimeout` in express.
 - `SAS_EXPRESS_MAX_BODY`: Set the size of request body payload, defaults to `100kb`
+- `SAS_EXPRESS_INJECTED_CONTROLLERS`: When set (_e.g. SAS_EXPRESS_INJECTED_CONTROLLERS=true_), automatically detects the available pallets in the chain's metadata and injects the appropriate controllers. If set to `false`, it uses the controllers defined in the chains configuration files. Defaults to `false` when the flag is not defined. Note: This flag does not replace completely the need of the chains-config files since we are still retrieving the options from that file. 
 
 ### Substrate node
 

src/SidecarConfig.ts

@@ -55,6 +55,7 @@ export class SidecarConfig {
 				PORT: config.Get(MODULES.EXPRESS, CONFIG.PORT) as number,
 				KEEP_ALIVE_TIMEOUT: config.Get(MODULES.EXPRESS, CONFIG.KEEP_ALIVE_TIMEOUT) as number,
 				MAX_BODY: config.Get(MODULES.EXPRESS, CONFIG.MAX_BODY) as string,
+				INJECTED_CONTROLLERS: config.Get(MODULES.EXPRESS, CONFIG.INJECTED_CONTROLLERS) as boolean,
 			},
 			SUBSTRATE: {
 				URL: config.Get(MODULES.SUBSTRATE, CONFIG.URL) as string,

src/Specs.ts

@@ -95,6 +95,19 @@ export class Specs {
 				type: 'string',
 			}),
 		);
+
+		this._specs.appendSpec(
+			MODULES.EXPRESS,
+			this._specs.getSpec(
+				CONFIG.INJECTED_CONTROLLERS,
+				'Use the controllers from the chain configuration or use controllers injected from pallets definitions',
+				{
+					default: 'false',
+					type: 'boolean',
+					regexp: /^true|false$/,
+				},
+			),
+		);
 	}
 
 	/**

src/chains-config/astarControllers.ts

@@ -35,7 +35,6 @@ export const astarControllers: ControllerConfig = {
 		'PalletsAssets',
 		'PalletsErrors',
 		'PalletsStorage',
-		'Paras',
 		'RuntimeCode',
 		'RuntimeMetadata',
 		'RuntimeSpec',

src/chains-config/bifrostControllers.ts

@@ -31,7 +31,6 @@ export const bifrostControllers: ControllerConfig = {
 		'NodeTransactionPool',
 		'NodeVersion',
 		'PalletsStorage',
-		'Paras',
 		'RuntimeCode',
 		'RuntimeMetadata',
 		'RuntimeSpec',

src/chains-config/bifrostPolkadotControllers.ts

@@ -31,7 +31,6 @@ export const bifrostPolkadotControllers: ControllerConfig = {
 		'NodeTransactionPool',
 		'NodeVersion',
 		'PalletsStorage',
-		'Paras',
 		'RuntimeCode',
 		'RuntimeMetadata',
 		'RuntimeSpec',

src/chains-config/calamariControllers.ts

@@ -35,7 +35,6 @@ export const calamariControllers: ControllerConfig = {
 		'NodeVersion',
 		'PalletsStakingProgress',
 		'PalletsStorage',
-		'Paras',
 		'RuntimeCode',
 		'RuntimeMetadata',
 		'RuntimeSpec',

src/chains-config/coretimeControllers.ts

@@ -22,23 +22,11 @@ import { initLRUCache, QueryFeeDetailsCache } from './cache';
  */
 export const coretimeControllers: ControllerConfig = {
 	controllers: [
-		'AccountsBalanceInfo',
-		'AccountsConvert',
-		'AccountsProxyInfo',
 		'Blocks',
 		'BlocksExtrinsics',
-		'BlocksTrace',
 		'BlocksRawExtrinsics',
 		'NodeNetwork',
 		'NodeVersion',
-		'PalletsConsts',
-		'PalletsErrors',
-		'PalletsEvents',
-		'PalletsNominationPools',
-		'PalletsOnGoingReferenda',
-		'PalletsStakingProgress',
-		'PalletsStakingValidators',
-		'PalletsStorage',
 		'RuntimeCode',
 		'RuntimeMetadata',
 		'RuntimeSpec',

src/chains-config/index.ts

@@ -15,6 +15,7 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import { ApiPromise } from '@polkadot/api';
+import { ISidecarConfig } from 'src/types/sidecar-config';
 
 import { controllers } from '../controllers';
 import AbstractController from '../controllers/AbstractController';
@@ -49,7 +50,7 @@ import { shidenControllers } from './shidenControllers';
 import { soraControllers } from './soraControllers';
 import { westendControllers } from './westendControllers';
 
-const specToControllerMap: { [x: string]: ControllerConfig } = {
+export const specToControllerMap: { [x: string]: ControllerConfig } = {
 	westend: westendControllers,
 	polkadot: polkadotControllers,
 	polymesh: polymeshControllers,
@@ -117,3 +118,39 @@ function getControllersFromConfig(api: ApiPromise, config: ControllerConfig) {
 		return acc;
 	}, [] as AbstractController<AbstractService>[]);
 }
+
+/**
+ * Return an array of instantiated controller instances based off of a `specName`.
+ * @param pallets pallets available to define controllers
+ * @param api ApiPromise to inject into controllers
+ * @param specName specName of chain to get options
+ */
+
+export const getControllersByPallets = (pallets: string[], api: ApiPromise, specName: string) => {
+	const controllersSet: AbstractController<AbstractService>[] = [];
+	const config = specToControllerMap?.[specName]?.options || defaultControllers?.options;
+
+	Object.values(controllers).forEach((controller) => {
+		if (controller.canInjectByPallets(pallets)) {
+			controllersSet.push(new controller(api, config));
+		}
+	});
+
+	return controllersSet;
+};
+
+export const getControllers = (
+	api: ApiPromise,
+	config: ISidecarConfig,
+	specName: string,
+): AbstractController<AbstractService>[] => {
+	if (config.EXPRESS.INJECTED_CONTROLLERS) {
+		return getControllersByPallets(
+			(api.registry.metadata.toJSON().pallets as unknown as Record<string, unknown>[]).map((p) => p.name as string),
+			api,
+			specName,
+		);
+	} else {
+		return getControllersForSpec(api, specName);
+	}
+};

src/chains-config/karuraControllers.ts

@@ -31,7 +31,6 @@ export const karuraControllers: ControllerConfig = {
 		'NodeTransactionPool',
 		'NodeVersion',
 		'PalletsStorage',
-		'Paras',
 		'RuntimeCode',
 		'RuntimeMetadata',
 		'RuntimeSpec',

src/chains-config/mantaControllers.ts

@@ -26,7 +26,6 @@ export const mantaControllers: ControllerConfig = {
 		'AccountsStakingInfo',
 		'AccountsStakingPayouts',
 		'AccountsValidate',
-		'AccountsVestingInfo',
 		'Blocks',
 		'BlocksExtrinsics',
 		'BlocksRawExtrinsics',
@@ -35,7 +34,6 @@ export const mantaControllers: ControllerConfig = {
 		'NodeVersion',
 		'PalletsStakingProgress',
 		'PalletsStorage',
-		'Paras',
 		'RuntimeCode',
 		'RuntimeMetadata',
 		'RuntimeSpec',

src/controllers/AbstractController.ts

@@ -45,11 +45,21 @@ type SidecarRequestHandler =
 	| RequestHandler<IParaIdParam>
 	| RequestHandler;
 
+/*
+ * The required pallets for a controller can be a list of pallets in string, or a list of list of pallets, which represents an OR statement.
+ * Example:
+ * 		- [['pallet1', 'pallet2']] => Requires pallet1 AND pallet2
+ *      - [['pallet1', 'pallet2'], ['pallet1', 'pallet3']] => Requires (pallet1 AND pallet2) OR (pallet1 AND pallet3)
+ */
+export type RequiredPallets = string[][];
+
 /**
  * Abstract base class for creating controller classes.
  */
 export default abstract class AbstractController<T extends AbstractService> {
 	private _router: Router = express.Router();
+	static controllerName: string;
+	static requiredPallets: RequiredPallets;
 
 	constructor(
 		protected api: ApiPromise,
@@ -266,4 +276,20 @@ export default abstract class AbstractController<T extends AbstractService> {
 	static sanitizedSend<T>(res: Response<AnyJson>, body: T, options: ISanitizeOptions = {}): void {
 		res.send(sanitizeNumbers(body, options));
 	}
+
+	static canInjectByPallets(availablePallets: string[]): boolean {
+		if (!this.requiredPallets) {
+			return true;
+		}
+		if (this.requiredPallets.length === 1) {
+			if (this.requiredPallets[0].length === 0) {
+				return true;
+			}
+			return this.requiredPallets[0].every((pallet) => availablePallets.includes(pallet));
+		} else if (this.requiredPallets.length > 1) {
+			return this.requiredPallets.some((pallets) => pallets.every((p) => availablePallets.includes(p)));
+		}
+		// If requiredPallets is empty, then we can inject
+		return true;
+	}
 }

src/controllers/accounts/AccountsAssetsController.ts

@@ -66,11 +66,13 @@ import AbstractController from '../AbstractController';
  *
  */
 export default class AccountsAssetsController extends AbstractController<AccountsAssetsService> {
+	static controllerName = 'AccountsAssets';
+	static requiredPallets = [['Assets']];
+
 	constructor(api: ApiPromise) {
 		super(api, '/accounts/:address', new AccountsAssetsService(api));
 		this.initRoutes();
 	}
-
 	protected initRoutes(): void {
 		this.router.use(this.path, validateAddress);
 

src/controllers/accounts/AccountsBalanceInfoController.ts

@@ -56,6 +56,8 @@ import AbstractController from '../AbstractController';
  * - `BalanceLock`: https://crates.parity.io/pallet_balances/struct.BalanceLock.html
  */
 export default class AccountsBalanceController extends AbstractController<AccountsBalanceInfoService> {
+	static controllerName = 'AccountsBalanceInfo';
+	static requiredPallets = [['Balances', 'System']];
 	constructor(api: ApiPromise) {
 		super(api, '/accounts/:address/balance-info', new AccountsBalanceInfoService(api));
 		this.initRoutes();

src/controllers/accounts/AccountsCompareController.ts

@@ -24,6 +24,8 @@ import { ICompareQueryParams } from '../../types/requests';
 import AbstractController from '../AbstractController';
 
 export default class AccountsCompareController extends AbstractController<AccountsCompareService> {
+	static controllerName = 'AccountsCompare';
+	static requiredPallets = [];
 	constructor(api: ApiPromise) {
 		super(api, '/accounts/compare', new AccountsCompareService(api));
 		this.initRoutes();

src/controllers/accounts/AccountsConvertController.ts

@@ -24,6 +24,8 @@ import { IAddressParam, IConvertQueryParams } from '../../types/requests';
 import AbstractController from '../AbstractController';
 
 export default class AccountsConvertController extends AbstractController<AccountsConvertService> {
+	static controllerName = 'AccountsConvert';
+	static requiredPallets = [];
 	constructor(api: ApiPromise) {
 		super(api, '/accounts/:address/convert', new AccountsConvertService(api));
 		this.initRoutes();

src/controllers/accounts/AccountsPoolAssetsController.ts

@@ -66,6 +66,8 @@ import AbstractController from '../AbstractController';
  *
  */
 export default class AccountsPoolAssetsController extends AbstractController<AccountsPoolAssetsService> {
+	static controllerName = 'AccountsPoolAssets';
+	static requiredPallets = [['PoolAssets', 'Assets']];
 	constructor(api: ApiPromise) {
 		super(api, '/accounts/:address', new AccountsPoolAssetsService(api));
 		this.initRoutes();

src/controllers/accounts/AccountsProxyInfoController.ts

@@ -23,6 +23,8 @@ import { AccountsProxyInfoService } from '../../services';
 import AbstractController from '../AbstractController';
 
 export default class AccountsProxyInfoController extends AbstractController<AccountsProxyInfoService> {
+	static controllerName = 'AccountsProxyInfo';
+	static requiredPallets = [['Proxy']];
 	constructor(api: ApiPromise) {
 		super(api, '/accounts/:address/proxy-info', new AccountsProxyInfoService(api));
 		this.initRoutes();

src/controllers/accounts/AccountsStakingInfoController.ts

@@ -70,6 +70,13 @@ import AbstractController from '../AbstractController';
  * - `StakingLedger`: https://crates.parity.io/pallet_staking/struct.StakingLedger.html
  */
 export default class AccountsStakingInfoController extends AbstractController<AccountsStakingInfoService> {
+	static controllerName = 'AccountsStakingInfo';
+	static requiredPallets = [
+		['Staking', 'System'],
+		['ParachainStaking', 'ParachainSystem'],
+		['ParachainStaking', 'System'],
+		['Staking', 'ParachainSystem'],
+	];
 	constructor(api: ApiPromise) {
 		super(api, '/accounts/:address/staking-info', new AccountsStakingInfoService(api));
 		this.initRoutes();

src/controllers/accounts/AccountsStakingPayoutsController.ts

@@ -80,6 +80,13 @@ import AbstractController from '../AbstractController';
  *
  */
 export default class AccountsStakingPayoutsController extends AbstractController<AccountsStakingPayoutsService> {
+	static controllerName = 'AccountsStakingPayouts';
+	static requiredPallets = [
+		['Staking', 'System'],
+		['ParachainStaking', 'ParachainSystem'],
+		['ParachainStaking', 'System'],
+		['Staking', 'ParachainSystem'],
+	];
 	constructor(api: ApiPromise) {
 		super(api, '/accounts/:address/staking-payouts', new AccountsStakingPayoutsService(api));
 		this.initRoutes();

src/controllers/accounts/AccountsValidateController.ts

@@ -21,6 +21,8 @@ import { AccountsValidateService } from '../../services/accounts';
 import AbstractController from '../AbstractController';
 
 export default class ValidateAddressController extends AbstractController<AccountsValidateService> {
+	static controllerName = 'AccountsValidate';
+	static requiredPallets = [];
 	constructor(api: ApiPromise) {
 		super(api, '/accounts/:address/validate', new AccountsValidateService(api));
 		this.initRoutes();

src/controllers/accounts/AccountsVestingInfoController.ts

@@ -44,6 +44,9 @@ import AbstractController from '../AbstractController';
  * - `VestingInfo`: https://crates.parity.io/pallet_vesting/struct.VestingInfo.html
  */
 export default class AccountsVestingInfoController extends AbstractController<AccountsVestingInfoService> {
+	static controllerName = 'AccountsVestingInfo';
+	static requiredPallets = [['Vesting'], ['CalamariVesting']];
+
 	constructor(api: ApiPromise) {
 		super(api, '/accounts/:address/vesting-info', new AccountsVestingInfoService(api));
 		this.initRoutes();

src/controllers/blocks/BlocksController.ts

@@ -104,6 +104,8 @@ import AbstractController from '../AbstractController';
  */
 export default class BlocksController extends AbstractController<BlocksService> {
 	private blockStore: LRUCache<string, IBlock>;
+	static controllerName = 'Blocks';
+	static requiredPallets = [['System', 'Session']];
 	constructor(
 		api: ApiPromise,
 		private readonly options: ControllerOptions,

src/controllers/blocks/BlocksExtrinsicsController.ts

@@ -26,6 +26,8 @@ import AbstractController from '../AbstractController';
 
 export default class BlocksExtrinsicsController extends AbstractController<BlocksService> {
 	private blockStore: LRUCache<string, IBlock>;
+	static controllerName = 'BlocksExtrinsics';
+	static requiredPallets = [['System', 'Session']];
 	constructor(api: ApiPromise, options: ControllerOptions) {
 		super(
 			api,

src/controllers/blocks/BlocksRawExtrinsicsController.ts

@@ -23,6 +23,8 @@ import { INumberParam, IRequestHandlerWithMetrics } from '../../types/requests';
 import AbstractController from '../AbstractController';
 
 export default class BlocksRawExtrinsicsController extends AbstractController<BlocksService> {
+	static controllerName = 'BlocksRawExtrinsics';
+	static requiredPallets = [];
 	constructor(api: ApiPromise, options: ControllerOptions) {
 		super(
 			api,

src/controllers/blocks/BlocksTraceController.ts

@@ -23,6 +23,8 @@ import AbstractController from '../AbstractController';
 import BlocksController from './BlocksController';
 
 export default class BlocksTraceController extends AbstractController<BlocksTraceService> {
+	static controllerName = 'BlocksTrace';
+	static requiredPallets = [];
 	constructor(api: ApiPromise) {
 		super(api, '/experimental/blocks', new BlocksTraceService(api));
 		this.initRoutes();

src/controllers/contracts/ContractsInkController.ts

@@ -25,6 +25,8 @@ import { IBodyContractMetadata, IContractQueryParam, IPostRequestHandler } from
 import AbstractController from '../AbstractController';
 
 export default class ContractsInkController extends AbstractController<ContractsInkService> {
+	static controllerName = 'ContractsInk';
+	static requiredPallets = [];
 	constructor(api: ApiPromise) {
 		super(api, '/contracts/ink/:address', new ContractsInkService(api));
 		this.initRoutes();