Skip to content
This repository has been archived by the owner on Nov 15, 2022. It is now read-only.

Commit

Permalink
Merge pull request #84 from Colm3na/ana/fix-staking-crawler-for-update
Browse files Browse the repository at this point in the history 
Ana/fix the staking crawler
  • Loading branch information
@mariopino
mariopino authored Mar 17, 2020
2 parents b18c3db + 9b90f37 commit c162237
Showing 1 changed file with 65 additions and 30 deletions.
95 changes: 65 additions & 30 deletions lib/crawlers/staking.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,16 +32,21 @@ module.exports = {
// Get current session info
const sessionInfo = await api.derive.session.info();

// Retrieve the active era
let activeEra = await api.query.staking.activeEra();
activeEra = JSON.parse(JSON.stringify(activeEra));
const currentEraIndex = activeEra.index;

if (sessionInfo.currentIndex > currentDBIndex) {
if (!crawlerIsRunning) {
await module.exports.storeStakingInfo(api, pool, blockNumber, sessionInfo);
await module.exports.storeStakingInfo(api, pool, blockNumber, sessionInfo, currentEraIndex);
}
}
});
},
storeStakingInfo: async function (api, pool, blockNumber, sessionInfo) {
storeStakingInfo: async function (api, pool, blockNumber, sessionInfo, currentEraIndex) {
crawlerIsRunning = true;

const currentIndex = sessionInfo.currentIndex.toNumber();
// console.log(`[PolkaStats backend v3] - Staking crawler - \x1b[32mCurrent session index is #${currentIndex}\x1b[0m`);

Expand All @@ -50,13 +55,29 @@ module.exports = {
//
// Get active validators, imOnline data, current elected and current era points earned
//
const [validators, imOnline, currentElected, currentEraPointsEarned] = await Promise.all([
const [validators, imOnline, exposures, erasRewardPoints] = await Promise.all([
api.query.session.validators(),
api.derive.imOnline.receivedHeartbeats(),
api.query.staking.currentElected(),
api.query.staking.currentEraPointsEarned()
api.query.staking.erasStakers.entries(currentEraIndex),
api.query.staking.erasRewardPoints(currentEraIndex)
]);


//
// Get all validator addresses in the last era
//
const eraExposures = exposures.map(([key, exposure]) => {
return {
accountId: key.args[1].toHuman(),
exposure: JSON.parse(JSON.stringify(exposure))
}
});
//
// Get all validator addresses in the last era
//
const eraValidatorList = eraExposures.map(exposure => {
return exposure.accountId;
});

//
// Map validator authorityId to staking info object
//
Expand All @@ -71,7 +92,7 @@ module.exports = {
validator.sessionIdHex = validator.sessionIds.length !== 0 ? validator.sessionIds.toHex() : ``;
validator.nextSessionIdHex = validator.nextSessionIds.length !== 0 ? validator.nextSessionIds.toHex() : ``;
})

//
// Add imOnline property to validator object
//
Expand All @@ -86,13 +107,13 @@ module.exports = {
//
for(let i = 0; i < validatorStaking.length; i++) {
let validator = validatorStaking[i];
if (Number.isInteger(currentElected.indexOf(validator.accountId))) {
if (Number.isInteger(eraValidatorList.indexOf(validator.accountId))) { // TODO: refactor duplicity with lines 171...
validator.currentElected = true;
} else {
validator.currentElected = false;
}
if (currentEraPointsEarned.individual[currentElected.indexOf(validator.accountId)]) {
validator.currentEraPointsEarned = currentEraPointsEarned.individual[currentElected.indexOf(validator.accountId)];
if (erasRewardPoints.individual[eraValidatorList.indexOf(validator.accountId)]) { // TODO: refactor
validator.erasRewardPoints = erasRewardPoints.individual[eraValidatorList.indexOf(validator.accountId)]; // TODO: refactor
}
}

Expand Down Expand Up @@ -128,8 +149,8 @@ module.exports = {
await pool.query(sql);

// populate validator_active table
console.log(`[PolkaStats backend v3] - Staking crawler - \x1b[33mPopulating validator_num_nominators table\x1b[0m`);
sql = `INSERT INTO validator_num_nominators (block_number, session_index, account_id, active, timestamp) VALUES ('${blockNumber}', '${currentIndex}', '${validator.accountId.toString()}', 'true', extract(epoch from now()));`;
console.log(`[PolkaStats backend v3] - Staking crawler - \x1b[33mPopulating validator_active table\x1b[0m`);
sql = `INSERT INTO validator_active (block_number, session_index, account_id, active, timestamp) VALUES ('${blockNumber}', '${currentIndex}', '${validator.accountId.toString()}', 'true', extract(epoch from now()));`;
await pool.query(sql);

})
Expand All @@ -142,20 +163,27 @@ module.exports = {

// TODO: Replace queries, check https://github.com/Colm3na/polkastats-backend-v3/pull/63#issuecomment-598613801

// console.log(`[PolkaStats backend v3] - Staking crawler - \x1b[33mPopulating validator_era_points table\x1b[0m`);
// const lastSessionIndex = sessionInfo.currentIndex - 1;
// const lastSessionBlockNumber = blockNumber - sessionInfo.sessionProgress - 1;
// const lastSessionBlockHash = await api.rpc.chain.getBlockHash(lastSessionBlockNumber);
// const lastSessionEraPoints = await api.query.staking.currentEraPointsEarned.at(lastSessionBlockHash);
// const lastSessionElected = await api.query.staking.currentElected.at(lastSessionBlockHash);

// validatorStaking.forEach(async validator => {
// if (lastSessionEraPoints.individual[lastSessionElected.indexOf(validator.accountId)]) {
// const eraPoints = lastSessionEraPoints.individual[lastSessionElected.indexOf(validator.accountId)];
// sql = `INSERT INTO validator_era_points (block_number, session_index, account_id, era_points, timestamp) VALUES ('${lastSessionBlockNumber}', '${lastSessionIndex}', '${validator.accountId.toString()}', '${eraPoints}', extract(epoch from now()));`;
// await pool.query(sql);
// }
// })
console.log(`[PolkaStats backend v3] - Staking crawler - \x1b[33mPopulating validator_era_points table\x1b[0m`);

const lastSessionBlockNumber = blockNumber - sessionInfo.sessionProgress - 1;
// const lastSessionEraPoints = await api.query.staking.currentEraPointsEarned.at(lastSessionBlockHash); // I guess this is the same than eraRewardsPoints?
// const lastSessionElected = await api.query.staking.currentElected.at(lastSessionBlockHash); // And this like the elected validators from exposures
const eraEraPoints = await api.query.staking.erasRewardPoints(currentEraIndex); // TODO: avoid duplicates
const eraEraPointsList = eraEraPoints.individual.toHuman();

validatorStaking.forEach(async validator => {
const validatorAccountId = validator.accountId.toHuman();
if (eraEraPointsList[validatorAccountId]) {
const validatorEraPoints = parseInt(eraEraPointsList[validatorAccountId])
let sqlInsert = `INSERT INTO validator_era_points (block_number, session_index, account_id, era_points, timestamp) VALUES ('${lastSessionBlockNumber}', '${currentIndex}', '${validatorAccountId}', '${validatorEraPoints}', extract(epoch from now()));`;
try {
const res = await pool.query(sqlInsert);
// console.log(`[PolkaStats backend v3] - Staking crawler - \x1b[33mResponse from Database is ${JSON.stringify(res)}]`)
} catch (error) {
console.log(`[PolkaStats backend v3] - Staking crawler - \x1b[31mERROR: ${JSON.stringify(error)}\x1b[0m`);
}
}
})

//
// Fetch intention validators
Expand Down Expand Up @@ -183,13 +211,20 @@ module.exports = {
intention.nextSessionIdHex = intention.nextSessionIds.toHex();
}
}



//
// Populate intention_staking table
//

console.log(`[PolkaStats backend v3] - Staking crawler - \x1b[33mPopulating intention_staking table\x1b[0m`);

if (intentionStaking) {
let sqlInsert = `INSERT INTO intention_staking (block_number, session_index, json, timestamp) VALUES ('${blockNumber}', '${currentIndex}', '${JSON.stringify(intentionStaking)}', extract(epoch from now()));`;
try {
await pool.query(sqlInsert);
const res = await pool.query(sqlInsert);
// console.log(`[PolkaStats backend v3] - Staking crawler - \x1b[33mResponse from Database is ${JSON.stringify(res)}]`)
} catch (error) {
// console.log(`[PolkaStats backend v3] - Staking crawler - \x1b[31mSQL: ${sqlInsert}\x1b[0m`);
console.log(`[PolkaStats backend v3] - Staking crawler - \x1b[31mERROR: ${JSON.stringify(error)}\x1b[0m`);
}
}
Expand Down

0 comments on commit c162237

Please sign in to comment.