From c3ccb4be4123b51647aef2af6b2caf7e89ee5ba0 Mon Sep 17 00:00:00 2001 From: Ynon Flum Date: Thu, 15 Dec 2022 19:37:38 +0200 Subject: [PATCH 01/18] Use 0-based indices in threshsign --- .../preprocessor/tests/preprocessor_test.cpp | 285 +++++------------- .../threshsign/eddsa/EdDSAMultisigFactory.cpp | 14 +- .../threshsign/eddsa/EdDSAMultisigVerifier.h | 1 + 3 files changed, 87 insertions(+), 213 deletions(-) diff --git a/bftengine/src/preprocessor/tests/preprocessor_test.cpp b/bftengine/src/preprocessor/tests/preprocessor_test.cpp index 322904131b..ea4e65f951 100644 --- a/bftengine/src/preprocessor/tests/preprocessor_test.cpp +++ b/bftengine/src/preprocessor/tests/preprocessor_test.cpp @@ -23,9 +23,14 @@ #include "ReplicaConfig.hpp" #include "IncomingMsgsStorageImp.hpp" #include "gtest/gtest.h" +#include "threshsign/eddsa/EdDSAMultisigFactory.h" +#include "CryptoManager.hpp" #include "tests/config/test_comm_config.hpp" #include "bftengine/MsgsCommunicator.hpp" +#include +#include + using namespace std; using namespace bft::communication; using namespace bftEngine; @@ -56,7 +61,6 @@ const NodeIdType replica_1 = 1; const NodeIdType replica_2 = 2; const NodeIdType replica_3 = 3; const NodeIdType replica_4 = 4; -const NodeIdType replica_5 = 5; const ViewNum viewNum = 1; PreProcessorRecorder preProcessorRecorder; std::shared_ptr sdm = make_shared(); @@ -65,7 +69,8 @@ uint64_t reqRetryId = 20; ReplicaConfig& replicaConfig = ReplicaConfig::instance(); char buf[bufLen]; -std::unique_ptr sigManager[numOfReplicas_4]; +std::shared_ptr sigManager[numOfReplicas_4]; +std::shared_ptr cryptoManager[numOfReplicas_4]; std::unique_ptr replicasInfo[numOfReplicas_4]; class DummyRequestsHandler : public IRequestsHandler { @@ -161,159 +166,7 @@ class DummyPreProcessor : public PreProcessor { }; // clang-format off -unordered_map replicaRSAPrivKeys = { - {replica_0, "308204BA020100300D06092A864886F70D0101010500048204A4308204A00201000282010100C55B8F7979BF24B335017082BF33EE2960E3A0" - "68DCDB45CA3017214BFB3F32649400A2484E2108C7CD07AA7616290667AF7C7A1922C82B51CA01867EED9B60A57F5B6EE33783EC258B234748" - "8B0FA3F99B05CFFBB45F80960669594B58C993D07B94D9A89ED8266D9931EAE70BB5E9063DEA9EFAF744393DCD92F2F5054624AA048C7EE50B" - "EF374FCDCE1C8CEBCA1EF12AF492402A6F56DC9338834162F3773B119145BF4B72672E0CF2C7009EBC3D593DFE3715D942CA8749771B484F72" - "A2BC8C89F86DB52ECC40763B6298879DE686C9A2A78604A503609BA34B779C4F55E3BEB0C26B1F84D8FC4EB3C79693B25A77A158EF88292D4C" - "01F99EFE3CC912D09B020111028201001D05EF73BF149474B4F8AEA9D0D2EE5161126A69C6203EF8162184E586D4967833E1F9BF56C89F68AD" - "35D54D99D8DB4B7BB06C4EFD95E840BBD30C3FD7A5E890CEF6DB99E284576EEED07B6C8CEBB63B4B80DAD2311D1A706A5AC95DE768F017213B" - "896B9EE38D2E3C2CFCE5BDF51ABD27391761245CDB3DCB686F05EA2FF654FA91F89DA699F14ACFA7F0D8030F74DBFEC28D55C902A27E9C03AB" - "1CA2770EFC5BE541560D86FA376B1A688D92124496BB3E7A3B78A86EBF1B694683CDB32BC49431990A18B570C104E47AC6B0DE5616851F4309" - "CFE7D0E20B17C154A3D85F33C7791451FFF73BFC4CDC8C16387D184F42AD2A31FCF545C3F9A498FAAC6E94E902818100F40CF9152ED4854E1B" - "BF67C5EA185C52EBEA0C11875563AEE95037C2E61C8D988DDF71588A0B45C23979C5FBFD2C45F9416775E0A644CAD46792296FDC68A98148F7" - "BD3164D9A5E0D6A0C2DF0141D82D610D56CB7C53F3C674771ED9ED77C0B5BF3C936498218176DC9933F1215BC831E0D41285611F512F68327E" - "4FBD9E5C5902818100CF05519FD69D7C6B61324F0A201574C647792B80E5D4D56A51CF5988927A1D54DF9AE4EA656AE25961923A0EC046F1C5" - "69BAB53A64EB0E9F5AB2ABF1C9146935BA40F75E0EB68E0BE4BC29A5A0742B59DF5A55AB028F1CCC42243D2AEE4B74344CA33E72879EF2D1CD" - "D874A7F237202AC7EB57AEDCBD539DEFDA094476EAE613028180396C76D7CEC897D624A581D43714CA6DDD2802D6F2AAAE0B09B885974533E5" - "14D6167505C620C51EA41CA70E1D73D43AA5FA39DA81799922EB3173296109914B98B2C31AAE515434E734E28ED31E8D37DA99BA11C2E693B6" - "398570ABBF6778A33C0E40CC6007E23A15C9B1DE6233B6A25304B91053166D7490FCD26D1D8EAC5102818079C6E4B86020674E392CA6F6E5B2" - "44B0DEBFBF3CC36E232F7B6AE95F6538C5F5B0B57798F05CFD9DFD28D6DB8029BB6511046A9AD1F3AE3F9EC37433DFB1A74CC7E9FAEC08A79E" - "D9D1D8187F8B8FA107B08F7DAFE3633E1DCC8DC9A0C8689EB55A41E87F9B12347B6A06DB359D89D6AFC0E4CA2A9FF6E5E46EF8BA2845F39665" - "0281802A89B2BD4A665A0F07DCAFA6D9DB7669B1D1276FC3365173A53F0E0D5F9CB9C3E08E68503C62EA73EB8E0DA42CCF6B136BF4A85B0AC4" - "24730B4F3CAD8C31D34DD75EF2A39B6BCFE3985CCECC470CF479CF0E9B9D6C7CE1C6C70D853728925326A22352DF73B502D4D3CBC2A770DE27" - "6E1C5953DF7A9614C970C94D194CAE9188"}, - {replica_1, "308204BC020100300D06092A864886F70D0101010500048204A6308204A20201000282010100CAF26C09B1F6F056F66B54AA0EA002EA236151" - "3A721A58E44472ADE74DBD833890ED9EE8E0CFF1B84C3DFE9E6226C8CEBFFEDFE71A6BCFE5A599D02FD9940997E864D400BF440BA7B1A568A5" - "ACF7688B025B5B207E5651E597141FAEF733FA9D87732B8FBA31E082BE777AEB992B00873764655C3B1F8FBF8DD2446F4F9F79883E21DBC1A1" - "7431E42AF01B9D4DE54B9880E7FB397655C79D7C3A5981CA64A32E2F05FCF0B127F4C6D349056276992B7403D36E03D14C45B6F5F66786280C" - "921236A17EE0F311821BDF925A47DCA590BB31C60646692014317BE0DD32248DF557A77F80F336C82461B659266F74F495BA5CE194043505B5" - "F0C363263BF73BB897020111028201005F8123C853BF8028EC6EBE6E2500015F1FB55366CC48A24D4D6324A915865BDE6251B4315ABC3583E7" - "A4B40E4C4E7C9D8786FFF448AB34A84DEE079E0C096DED221154B50EB69C12ADF37C8A334740416A85580F4A82F95CFBCD3C1619FA57D1A927" - "238EEE3596D41D656705754169A90B021194D08752B47EF9899DCB1DDED5DCDDA688D9FA88F9303E99CAB040D2E47DCE866717580C2CD01DB9" - "4E8FE215EAA254432E3B399B90260255DEE7FADED4643814C0DE41DE237BBB7173CFF16AC9237AADA4595FD95EB92E30D2127C7761C6A1CD75" - "C1F12F5B92B6602E1CDC06E112321B0A41C8A102D785B7C4C7D2441DA983346B81873984DD680FB4B1C9AD7102818100D5E41BE52CECFB7405" - "01B87C631DEFB3DD855EE95553A816BE8D3DFAE200086E1C12E9FBDDD8737F2CCE2CED1D82431ADC63C1B34EA59C27E98283EB20EAF8F5A29D" - "45ACE2E5C3173F396DF4356D54F3ED0CF9BBA222E3B9194CD15F88B6AB8FAFDFACACCC97D87555CD1E864D09DF795FB8BE3F9BE2CB8E52F641" - "FCEDC8E97902818100F2E6BDF9A552D35E9F695C52343D9BBF180BBEB50F6705A7836DF1BFF6A42C2D7A000432957516B555B5E1FBAC21CED5" - "D2788036AA5AB183A5859284ED409631289F8836D240111B56D6C4953FEFBE177EA137F08ADCABD5CAD07F709E83BB29B0F55AD09E65F5C656" - "8FE166FF4BE581F4F2066025E3902819EFC2DF0FA63E8F0281803EE8BCE90D36A44F4CC44551C2CC91CB7D637644A0A022610ADE3F67E81E20" - "98DB149F2BF5F45E347696FE279F446E16F586C080081297570871AE5436DBB2A2993D50BA60DA2A5221A77AB13CE3EBCF45B885AFA8286118" - "52BC3D94919F23667F058D23C3B4309AFB1E36278011F66EFE0928E58833A547FA486DC2DC8662C902818100AB75B346CF0D49E870869B8552" - "0D5EE13E26687FCEA3130CD53E8C8780EC5B6B652D3023B4CB1F1696DABDA2979F64D32B27E208784004D565C7B2B82F006A0495255117A378" - "848BC4D3D60EFFF4862EB3BD186D8F325B2D801AB44F7EF3932C7CE96D47F75707D74C2953D03BBD1A79DA1440BC56FAFC588AC75C6138391D" - "1902818100BBC05F561AEF4BDE1ECABB5CD313E6173F5E46AC85B4462A8CA689E4E84B95E66733081518DB01714B7B18E44B780DB5A6DA2612" - "506F46E4D91C8CB582AFD54C7509C9947EC754BFDD0D35FAECE2E729DBD21A62E33C3DEF556913ECF4C8D75910E860D36AFD0977FF9114ACEA" - "8F44882259C6F1D8314B653F64F4655CFD68A6"}, - {replica_2, "308204BA020100300D06092A864886F70D0101010500048204A4308204A00201000282010100DA68B2830103028D0ECD911D20E732257F8A39" - "BC5214B670C8F1F836CD1D88E6421ABD3C47F16E37E07DCFE89FC11ECBED73896FC978B7E1519F59318F3E24BB5C8962B0EA44889C3A81021C" - "C9E8EAA94099B9305B0F7731C27CC528CFEE710066CB69229F8AA04DBD842FA8C3143CB0EF1164E45E34B114FBA610DBDFC7453DAD995743AD" - "1489AA131D0C730163ECA7317048C1AD1008ED31A32CB773D94B5317CAAF95674582DCA1AC9FEBE02079836B52E1E3C8D39571018A7657650D" - "52D2DF3363F88A690628D64DCBECB88BBA432F27C32E7BC5BC0F1D16D0A027D1D40362F8CEDBE8BED05C2BCE96F33A75CD70014626B32E5EE8" - "1EBD3116F6F1E5231B02011102820100201E749ACB716241EB96B375398B6941BFEEAE23393F480186F668444B572AB873220CC519A3812655" - "B8261AAE14DEE1C1097617F7FB2A199B0FE7783AB650B22432524731828C8F7203E9B8F08422824D43C868FE55190ED8D61CFE78EE5BE97887" - "5339CC2AF974D81AF7F32BBF361A050A165DD19E5646D9B68A02377F2FD417BE92D2722CEE33B3A0F7B111639C092B4024BB08ACEBE9F9BC28" - "C14BB611DBE627136B568493C6995D81442F09DFEFBA4378A34DEF28F3BD862F4C28BECA87A95551FC2DB48766F82752C7E8B34E8599590190" - "B12A648AC1E361FE28A3253EC615381A066BFD351C18F0B236429A950106DD4F1134A3DE6CEB1094B7CE5C7102818100EA51580294BB0FD7DA" - "FDC00364A9B3C4781F70C4BBE8BDE0D82895ADE549E25388E78A53EF83CF9A3D9528F70CB5A57A9A187D5AC68682F03B26C2F6A409E1554FED" - "061BDC944985102F2988A5A8CD639BA7890FD689D7171612159AAA818F5502EF80D5D13339826A98914A321B5B4512EC24AF3EE80F982D3418" - "0C85B3E38102818100EE9E7F43A98E593F72A584EEC014E0A46003116B82F5D3A21DAE4EB3F21FBC5B71D96E012B6F5FFBEACED4BEC6C147AA" - "AB6F9698F0592F3A8A6CD827ABF210497635639043D5F0B461E00914B152D6ECB3EFC9138A1B9FAEE09420AB9C2E1436B6AC36EEB8AD43709B" - "BFA0ED30FBF09C1A91BAB71410E49E11BE8E2A571C649B02818044EABF8849DCAA4E8BB40B4C4AC8802AB9EB212ACDDB0AAB8ADEC29C8EBB60" - "AF284419A0376300D3030DC0C121DB128D789DCA841C45AE0A6BC01B397B8A6F7371DC4D1740E051DBD79566919A2296C2F18BA0C86C46A8AC" - "6FE73387D7CBC0BEA682AD6C105A5C356AA557E8A553571450DC0ACA218F8C1DB2F1343FEB16CA71028180704A963DF57029FFBD7B11614B55" - "1E6B7879EA1479DD184C4A33E8CD26A585D0AE0BF7881470A5A3B9CABE77E50FA941419DEC8434DEACD04124297C14AE25C837A0A752F2BF07" - "DC6A4B4F91446337F6EB43A9EB13D0C39D96DC4B9C0D42DC55FB9C5615FC8DC5622B2D006F9E94AD76A31766ECBE26113B53A4F79B744998C1" - "028180124E2A5DAC7AC7017647FE1B6871484B7B38A3C060B992B871939C8A75BC35DB771F041910C0092E01B545E1910109AA827F34F56327" - "15E06819ADED60117FD8B9400C3F307EA022D5AC5394E65D28EF434DC068494D33ACF70B43791C1C13C35AC680E0038DB411ADEBBC067F477E" - "01E8CF793FB076AE47BD83B935B8041DC1"}, - {replica_3, "308204BB020100300D06092A864886F70D0101010500048204A5308204A10201000282010100B4D984129ECC2F4350596DD602EB5337B78E33" - "C4D945C70C2CACFF6237037BEF897D6893079D1067F07FF023D66C77BB41F6D41215C9912D995215C6F7651F1728AAB65CF20CBE81E5E7F202" - "E474AF535E92BBC386A7A496263BC4189FD9DD5E801C3023038EFE5B5DE35AA3F70C10F87259D586C36D3DF524E9DA2140C481365985AD185E" - "1CF3E78B6B82FDD274CCD1267838A0B91F9B48544AFB2C71B158A26E6154CECB43ECEE13FFBD6CA91D8D115B6B91C55F3A88E3F389E9A2AF4B" - "C381D21F9ABF2EA16F58773514249B03A4775C6A10561AFC8CF54B551A43FD014F3C5FE12D96AC5F117645E26D125DC7430114FA60577BF7C9" - "AA1224D190B2D8A83B020111028201004FC95FEA18E19C6176459256E32B95A7A3CDCB8B8D08322B04A6ACE790BDC5BC806C087D19F2782DDB" - "0B444C0BC6710ED9564E8073061A66F0D163F5E59D8DB764C3C8ECC523BD758B13815BA1064D597C8C078AF7A450241FED30DDAFEF2CF4FC48" - "ABD336469D648B4DB70C1A2AF86D9BDC56AC6546C882BD763A963329844BF0E8C79E5E7A5C227A1CDB8473965090084BEAC728E4F86670BAE0" - "5DA589FAE1EBC98A26BF207261EECE5A92759DE516306B0B2F715370586EF45447F82CC37206F70D762AAB75215482A67FE6742E9D644AB765" - "4F02476EAE15A742FCB2266DAE437DC76FB17E1698D4987945C779A4B89F5FB3F5E1B194EFA4024BA3797F2502818100E2BDA6F0031EAA027B" - "1E8099D9D2CF408B8C9D83A22BFBAAD3BCFA952469A001C25E5E2E89DAF28636633E8D7C5E1E5FD8384767DB5B384DAB38147EA46871D5E31E" - "DC110026E42E90C44F02110A369ACF0445C617CC25CD1207F116B9AF2FA78A89E6C6345EAFD0607CB145410DD6619AC0C034B54283A0065B93" - "3550B8DACF02818100CC2FDB49EB3E45DB2EB644048C3C35E3BB50980453DB8EB54DD5595CA2DBC43A2F2912D0F696E6126AFBF5D7777BABB2" - "6AC931030B8896343BC19EA30B8EEBFECE2617A2564B3D66E29DF66708254877CC33E699501A2BA4D0D7D02F068B1DCF6C7A0794F24BEE83BE" - "2E8453D3E436C3B59D2D9BEEB5B38541EF170544EA46D502818100AD63DA02D5359110F4BCF8EE1F0A9E7CA6F30F0A4ED6570A29726544DF9C" - "10F2495738F6696B31EE29972FD59B57082B2CDFBE223E54D0B3DD49009D144FDE94808102A396B454239BE169982B25ED85712162886C8D0D" - "D90DC9D67ACA3AABF8971E28F1EBCFEFDB95140F16D764EF3B947547AFD5E791D4B9915274108D5C07028180300B42A7FB1DB61574671F1020" - "FF1BBD1D03E7888C33A91B99D7D8CA80AC2E2BCEDC7CE5DFAB08F546596705858682C09198C03CF3A7AADF1D1E7FADE49A196921725FE9F62F" - "D236537076365C4501FE11EE182412D8FB35D6C95E292EB7524EEC58F2B9A26C381EFF92797D22CC491EFD8E6515A1942A3D78ECF65B97BEA7" - "410281806625E51F70BB9E00D0260941C56B22CDCECC7EA6DBC2C28A1CDA0E1AFAACD39C5E962E40FCFC0F5BBA57A7291A1BDC5D3D29495B45" - "CE2B3BEC7748E6C351FF934537BCA246CA36B840CF68BE6D473F7EE079B0EFDFF6B1BB554B06093E18B269FC3BD3F9876376CA7C673A93F142" - "7088BF0990AB8E232F269B5DBCD446385A66"}, - {replica_4, "308204BB020100300D06092A864886F70D0101010500048204A5308204A10201000282010100B4D984129ECC2F4350596DD602EB5337B78E33" - "C4D945C70C2CACFF6237037BEF897D6893079D1067F07FF023D66C77BB41F6D41215C9912D995215C6F7651F1728AAB65CF20CBE81E5E7F202" - "E474AF535E92BBC386A7A496263BC4189FD9DD5E801C3023038EFE5B5DE35AA3F70C10F87259D586C36D3DF524E9DA2140C481365985AD185E" - "1CF3E78B6B82FDD274CCD1267838A0B91F9B48544AFB2C71B158A26E6154CECB43ECEE13FFBD6CA91D8D115B6B91C55F3A88E3F389E9A2AF4B" - "C381D21F9ABF2EA16F58773514249B03A4775C6A10561AFC8CF54B551A43FD014F3C5FE12D96AC5F117645E26D125DC7430114FA60577BF7C9" - "AA1224D190B2D8A83B020111028201004FC95FEA18E19C6176459256E32B95A7A3CDCB8B8D08322B04A6ACE790BDC5BC806C087D19F2782DDB" - "0B444C0BC6710ED9564E8073061A66F0D163F5E59D8DB764C3C8ECC523BD758B13815BA1064D597C8C078AF7A450241FED30DDAFEF2CF4FC48" - "ABD336469D648B4DB70C1A2AF86D9BDC56AC6546C882BD763A963329844BF0E8C79E5E7A5C227A1CDB8473965090084BEAC728E4F86670BAE0" - "5DA589FAE1EBC98A26BF207261EECE5A92759DE516306B0B2F715370586EF45447F82CC37206F70D762AAB75215482A67FE6742E9D644AB765" - "4F02476EAE15A742FCB2266DAE437DC76FB17E1698D4987945C779A4B89F5FB3F5E1B194EFA4024BA3797F2502818100E2BDA6F0031EAA027B" - "1E8099D9D2CF408B8C9D83A22BFBAAD3BCFA952469A001C25E5E2E89DAF28636633E8D7C5E1E5FD8384767DB5B384DAB38147EA46871D5E31E" - "DC110026E42E90C44F02110A369ACF0445C617CC25CD1207F116B9AF2FA78A89E6C6345EAFD0607CB145410DD6619AC0C034B54283A0065B93" - "3550B8DACF02818100CC2FDB49EB3E45DB2EB644048C3C35E3BB50980453DB8EB54DD5595CA2DBC43A2F2912D0F696E6126AFBF5D7777BABB2" - "6AC931030B8896343BC19EA30B8EEBFECE2617A2564B3D66E29DF66708254877CC33E699501A2BA4D0D7D02F068B1DCF6C7A0794F24BEE83BE" - "2E8453D3E436C3B59D2D9BEEB5B38541EF170544EA46D502818100AD63DA02D5359110F4BCF8EE1F0A9E7CA6F30F0A4ED6570A29726544DF9C" - "10F2495738F6696B31EE29972FD59B57082B2CDFBE223E54D0B3DD49009D144FDE94808102A396B454239BE169982B25ED85712162886C8D0D" - "D90DC9D67ACA3AABF8971E28F1EBCFEFDB95140F16D764EF3B947547AFD5E791D4B9915274108D5C07028180300B42A7FB1DB61574671F1020" - "FF1BBD1D03E7888C33A91B99D7D8CA80AC2E2BCEDC7CE5DFAB08F546596705858682C09198C03CF3A7AADF1D1E7FADE49A196921725FE9F62F" - "D236537076365C4501FE11EE182412D8FB35D6C95E292EB7524EEC58F2B9A26C381EFF92797D22CC491EFD8E6515A1942A3D78ECF65B97BEA7" - "410281806625E51F70BB9E00D0260941C56B22CDCECC7EA6DBC2C28A1CDA0E1AFAACD39C5E962E40FCFC0F5BBA57A7291A1BDC5D3D29495B45" - "CE2B3BEC7748E6C351FF934537BCA246CA36B840CF68BE6D473F7EE079B0EFDFF6B1BB554B06093E18B269FC3BD3F9876376CA7C673A93F142" - "7088BF0990AB8E232F269B5DBCD446385A66"} -}; - -unordered_map replicaRSAPubKeys = { - {replica_0, "30820120300D06092A864886F70D01010105000382010D00308201080282010100C55B8F7979BF24B335017082BF33EE2960E3A068DCDB45CA" - "3017214BFB3F32649400A2484E2108C7CD07AA7616290667AF7C7A1922C82B51CA01867EED9B60A57F5B6EE33783EC258B2347488B0FA3F99B" - "05CFFBB45F80960669594B58C993D07B94D9A89ED8266D9931EAE70BB5E9063DEA9EFAF744393DCD92F2F5054624AA048C7EE50BEF374FCDCE" - "1C8CEBCA1EF12AF492402A6F56DC9338834162F3773B119145BF4B72672E0CF2C7009EBC3D593DFE3715D942CA8749771B484F72A2BC8C89F8" - "6DB52ECC40763B6298879DE686C9A2A78604A503609BA34B779C4F55E3BEB0C26B1F84D8FC4EB3C79693B25A77A158EF88292D4C01F99EFE3C" - "C912D09B020111"}, - {replica_1, "30820120300D06092A864886F70D01010105000382010D00308201080282010100CAF26C09B1F6F056F66B54AA0EA002EA2361513A721A58E4" - "4472ADE74DBD833890ED9EE8E0CFF1B84C3DFE9E6226C8CEBFFEDFE71A6BCFE5A599D02FD9940997E864D400BF440BA7B1A568A5ACF7688B02" - "5B5B207E5651E597141FAEF733FA9D87732B8FBA31E082BE777AEB992B00873764655C3B1F8FBF8DD2446F4F9F79883E21DBC1A17431E42AF0" - "1B9D4DE54B9880E7FB397655C79D7C3A5981CA64A32E2F05FCF0B127F4C6D349056276992B7403D36E03D14C45B6F5F66786280C921236A17E" - "E0F311821BDF925A47DCA590BB31C60646692014317BE0DD32248DF557A77F80F336C82461B659266F74F495BA5CE194043505B5F0C363263B" - "F73BB897020111"}, - {replica_2, "30820120300D06092A864886F70D01010105000382010D00308201080282010100DA68B2830103028D0ECD911D20E732257F8A39BC5214B670" - "C8F1F836CD1D88E6421ABD3C47F16E37E07DCFE89FC11ECBED73896FC978B7E1519F59318F3E24BB5C8962B0EA44889C3A81021CC9E8EAA940" - "99B9305B0F7731C27CC528CFEE710066CB69229F8AA04DBD842FA8C3143CB0EF1164E45E34B114FBA610DBDFC7453DAD995743AD1489AA131D" - "0C730163ECA7317048C1AD1008ED31A32CB773D94B5317CAAF95674582DCA1AC9FEBE02079836B52E1E3C8D39571018A7657650D52D2DF3363" - "F88A690628D64DCBECB88BBA432F27C32E7BC5BC0F1D16D0A027D1D40362F8CEDBE8BED05C2BCE96F33A75CD70014626B32E5EE81EBD3116F6" - "F1E5231B020111"}, - {replica_3, "30820120300D06092A864886F70D01010105000382010D00308201080282010100B4D984129ECC2F4350596DD602EB5337B78E33C4D945C70C" - "2CACFF6237037BEF897D6893079D1067F07FF023D66C77BB41F6D41215C9912D995215C6F7651F1728AAB65CF20CBE81E5E7F202E474AF535E" - "92BBC386A7A496263BC4189FD9DD5E801C3023038EFE5B5DE35AA3F70C10F87259D586C36D3DF524E9DA2140C481365985AD185E1CF3E78B6B" - "82FDD274CCD1267838A0B91F9B48544AFB2C71B158A26E6154CECB43ECEE13FFBD6CA91D8D115B6B91C55F3A88E3F389E9A2AF4BC381D21F9A" - "BF2EA16F58773514249B03A4775C6A10561AFC8CF54B551A43FD014F3C5FE12D96AC5F117645E26D125DC7430114FA60577BF7C9AA1224D190" - "B2D8A83B020111"}, - {replica_4, "30820120300D06092A864886F70D01010105000382010D00308201080282010100B4D984129ECC2F4350596DD602EB5337B78E33C4D945C70C" - "2CACFF6237037BEF897D6893079D1067F07FF023D66C77BB41F6D41215C9912D995215C6F7651F1728AAB65CF20CBE81E5E7F202E474AF535E" - "92BBC386A7A496263BC4189FD9DD5E801C3023038EFE5B5DE35AA3F70C10F87259D586C36D3DF524E9DA2140C481365985AD185E1CF3E78B6B" - "82FDD274CCD1267838A0B91F9B48544AFB2C71B158A26E6154CECB43ECEE13FFBD6CA91D8D115B6B91C55F3A88E3F389E9A2AF4BC381D21F9A" - "BF2EA16F58773514249B03A4775C6A10561AFC8CF54B551A43FD014F3C5FE12D96AC5F117645E26D125DC7430114FA60577BF7C9AA1224D190" - "B2D8A83B020111"}, - {replica_5, "30820120300D06092A864886F70D01010105000382010D00308201080282010100B4D984129ECC2F4350596DD602EB5337B78E33C4D945C70C" - "2CACFF6237037BEF897D6893079D1067F07FF023D66C77BB41F6D41215C9912D995215C6F7651F1728AAB65CF20CBE81E5E7F202E474AF535E" - "92BBC386A7A496263BC4189FD9DD5E801C3023038EFE5B5DE35AA3F70C10F87259D586C36D3DF524E9DA2140C481365985AD185E1CF3E78B6B" - "82FDD274CCD1267838A0B91F9B48544AFB2C71B158A26E6154CECB43ECEE13FFBD6CA91D8D115B6B91C55F3A88E3F389E9A2AF4BC381D21F9A" - "BF2EA16F58773514249B03A4775C6A10561AFC8CF54B551A43FD014F3C5FE12D96AC5F117645E26D125DC7430114FA60577BF7C9AA1224D190" - "B2D8A83B020111"} -}; - -unordered_map replicaEdDSAPrivKeys = { +unordered_map replicaPrivKeys = { {replica_0, "61498efe1764b89357a02e2887d224154006ceacf26269f8695a4af561453eef"}, {replica_1, "247a74ab3620ec6b9f5feab9ee1f86521da3fa2804ad45bb5bf2c5b21ef105bc"}, {replica_2, "fb539bc3d66deda55524d903da26dbec1f4b6abf41ec5db521e617c64eb2c341"}, @@ -321,7 +174,7 @@ unordered_map replicaEdDSAPrivKeys = { {replica_4, "f2f3d43da68329bfe31419636072e27cfd1a8fff259be4bfada667080eb00556"} }; -unordered_map replicaEdDSAPubKeys = { +unordered_map replicaPubKeys = { {replica_0, "386f4fb049a5d8bb0706d3793096c8f91842ce380dfc342a2001d50dfbc901f4"}, {replica_1, "3f9e7dbde90477c24c1bacf14e073a356c1eca482d352d9cc0b16560a4e7e469"}, {replica_2, "2311c6013ff657844669d8b803b2e1ed33fe06eed445f966a800a8fbb8d790e8"}, @@ -330,8 +183,30 @@ unordered_map replicaEdDSAPubKeys = { }; // clang-format on -unordered_map replicaPrivKeys; -unordered_map replicaPubKeys; + +class PPCryptoSystem : public Cryptosystem { + public: + PPCryptoSystem(NodeIdType id) : id_{id} {} + IThresholdVerifier *createThresholdVerifier(uint16_t threshold = 0) override { + vector publicKeys; + for (int i = 0; i < static_cast(replicaPubKeys.size()); i++) { + publicKeys.push_back(replicaPubKeys.at(i)); + } + return factory_.newVerifier(threshold, publicKeys.size(), "", publicKeys); + } + IThresholdSigner *createThresholdSigner() override { return factory_.newSigner(id_ + 1, replicaPrivKeys.at(id_).c_str()); } + + private: + NodeIdType id_; + EdDSAMultisigFactory factory_; +}; + + +void switchReplicaContext(NodeIdType id) { + LOG_INFO(GL, "Switching replica context to replica " << id); + SigManager::reset(sigManager[id]); + CryptoManager::reset(cryptoManager[id]); +} void setUpConfiguration_4() { replicaConfig.replicaId = replica_0; @@ -352,16 +227,16 @@ void setUpConfiguration_4() { for (auto i = 0; i < replicaConfig.numReplicas; i++) { replicaConfig.replicaId = i; replicasInfo[i] = std::make_unique(replicaConfig, true, true); - sigManager[i].reset(SigManager::initInTesting(i, - replicaPrivKeys[i], - replicaConfig.publicKeysOfReplicas, - concord::crypto::KeyFormat::HexaDecimalStrippedFormat, - nullptr, - concord::crypto::KeyFormat::HexaDecimalStrippedFormat, - *replicasInfo[i].get())); + sigManager[i] = SigManager::init(i, + replicaPrivKeys[i], + replicaConfig.publicKeysOfReplicas, + concord::crypto::KeyFormat::HexaDecimalStrippedFormat, + nullptr, + concord::crypto::KeyFormat::HexaDecimalStrippedFormat, + *replicasInfo[i].get()); + cryptoManager[i] = CryptoManager::init(std::make_unique(i)); } replicaConfig.replicaId = replica_0; - SigManager::instance(sigManager[replicaConfig.replicaId].get()); } void setUpConfiguration_7() { @@ -399,10 +274,10 @@ PreProcessReplyMsgSharedPtr preProcessNonPrimary(NodeIdType replicaId, const bftEngine::impl::ReplicasInfo& repInfo, ReplyStatus status, OperationResult opResult) { - SigManager::instance(sigManager[replicaId].get()); + switchReplicaContext(replicaId); auto preProcessReplyMsg = make_shared( replicaId, clientId, 0, reqSeqNum, reqRetryId, buf, bufLen, "", status, opResult, viewNum); - SigManager::instance(sigManager[repInfo.myId()].get()); + switchReplicaContext(repInfo.myId()); preProcessReplyMsg->validate(repInfo); return preProcessReplyMsg; } @@ -413,7 +288,15 @@ void clearDiagnosticsHandlers() { registrar.status.clear(); } -TEST(requestPreprocessingState_test, notEnoughRepliesReceived) { +class PreprocessingStateTest : public testing::Test { + public: + void SetUp() override { + switchReplicaContext(replica_0); + } + +}; + +TEST_F(PreprocessingStateTest, notEnoughRepliesReceived) { RequestProcessingState reqState(replicaConfig.replicaId, replicaConfig.numReplicas, "", @@ -433,7 +316,7 @@ TEST(requestPreprocessingState_test, notEnoughRepliesReceived) { clearDiagnosticsHandlers(); } -TEST(requestPreprocessingState_test, filterDuplication) { +TEST_F(PreprocessingStateTest, filterDuplication) { RequestProcessingState reqState(replicaConfig.replicaId, replicaConfig.numReplicas, "", @@ -459,7 +342,7 @@ TEST(requestPreprocessingState_test, filterDuplication) { clearDiagnosticsHandlers(); } -TEST(requestPreprocessingState_test, notEnoughErrorRepliesReceived) { +TEST_F(PreprocessingStateTest, notEnoughErrorRepliesReceived) { RequestProcessingState reqState(replicaConfig.replicaId, replicaConfig.numReplicas, "", @@ -480,7 +363,7 @@ TEST(requestPreprocessingState_test, notEnoughErrorRepliesReceived) { clearDiagnosticsHandlers(); } -TEST(requestPreprocessingState_test, changePrimaryBlockId) { +TEST_F(PreprocessingStateTest, changePrimaryBlockId) { memset(buf, '5', bufLen); uint64_t blockId = 0; uint64_t primaryBlockId = 420; @@ -511,7 +394,7 @@ TEST(requestPreprocessingState_test, changePrimaryBlockId) { clearDiagnosticsHandlers(); } -TEST(requestPreprocessingState_test, allRepliesReceivedButNotEnoughSameHashesCollected) { +TEST_F(PreprocessingStateTest, allRepliesReceivedButNotEnoughSameHashesCollected) { RequestProcessingState reqState(replicaConfig.replicaId, replicaConfig.numReplicas, "", @@ -532,7 +415,7 @@ TEST(requestPreprocessingState_test, allRepliesReceivedButNotEnoughSameHashesCol ConcordAssertEQ(reqState.definePreProcessingConsensusResult(), CANCEL); } -TEST(requestPreprocessingState_test, primarySucceededWhileNonPrimariesFailed_ForcedSuccess) { +TEST_F(PreprocessingStateTest, primarySucceededWhileNonPrimariesFailed_ForcedSuccess) { RequestProcessingState reqState(replicaConfig.replicaId, replicaConfig.numReplicas, "", @@ -552,7 +435,7 @@ TEST(requestPreprocessingState_test, primarySucceededWhileNonPrimariesFailed_For ConcordAssertEQ(reqState.definePreProcessingConsensusResult(), COMPLETE); } -TEST(requestPreprocessingState_test, enoughSameRepliesReceived) { +TEST_F(PreprocessingStateTest, enoughSameRepliesReceived) { RequestProcessingState reqState(replicaConfig.replicaId, replicaConfig.numReplicas, "", @@ -573,7 +456,7 @@ TEST(requestPreprocessingState_test, enoughSameRepliesReceived) { clearDiagnosticsHandlers(); } -TEST(requestPreprocessingState_test, enoughSameErrorRepliesReceived) { +TEST_F(PreprocessingStateTest, enoughSameErrorRepliesReceived) { RequestProcessingState reqState(replicaConfig.replicaId, replicaConfig.numReplicas, "", @@ -595,7 +478,7 @@ TEST(requestPreprocessingState_test, enoughSameErrorRepliesReceived) { clearDiagnosticsHandlers(); } -TEST(requestPreprocessingState_test, primaryReplicaPreProcessingIsDifferentThanOneThatPassedConsensus) { +TEST_F(PreprocessingStateTest, primaryReplicaPreProcessingIsDifferentThanOneThatPassedConsensus) { RequestProcessingState reqState(replicaConfig.replicaId, replicaConfig.numReplicas, "", @@ -616,7 +499,7 @@ TEST(requestPreprocessingState_test, primaryReplicaPreProcessingIsDifferentThanO clearDiagnosticsHandlers(); } -TEST(requestPreprocessingState_test, primaryReplicaDidNotCompletePreProcessingWhileNonPrimariesDid) { +TEST_F(PreprocessingStateTest, primaryReplicaDidNotCompletePreProcessingWhileNonPrimariesDid) { RequestProcessingState reqState(replicaConfig.replicaId, replicaConfig.numReplicas, "", @@ -637,7 +520,7 @@ TEST(requestPreprocessingState_test, primaryReplicaDidNotCompletePreProcessingWh ConcordAssertEQ(reqState.definePreProcessingConsensusResult(), COMPLETE); } -TEST(requestPreprocessingState_test, primaryReplicaDidNotCompleteErrorPreProcessingWhileNonPrimariesDid) { +TEST_F(PreprocessingStateTest, primaryReplicaDidNotCompleteErrorPreProcessingWhileNonPrimariesDid) { RequestProcessingState reqState(replicaConfig.replicaId, replicaConfig.numReplicas, "", @@ -658,7 +541,7 @@ TEST(requestPreprocessingState_test, primaryReplicaDidNotCompleteErrorPreProcess ConcordAssertEQ(reqState.definePreProcessingConsensusResult(), COMPLETE); } -TEST(requestPreprocessingState_test, validatePreProcessBatchRequestMsg) { +TEST_F(PreprocessingStateTest, validatePreProcessBatchRequestMsg) { bftEngine::impl::ReplicasInfo repInfo(replicaConfig, false, false); PreProcessReqMsgsList batch; @@ -703,14 +586,14 @@ TEST(requestPreprocessingState_test, validatePreProcessBatchRequestMsg) { } } -TEST(requestPreprocessingState_test, validatePreProcessBatchReplyMsg) { +TEST_F(PreprocessingStateTest, validatePreProcessBatchReplyMsg) { bftEngine::impl::ReplicasInfo repInfo(replicaConfig, false, false); PreProcessReplyMsgsList batch; uint overallRepliesSize = 0; const auto numOfMsgs = 3; const auto senderId = 2; - SigManager::instance(sigManager[senderId].get()); + switchReplicaContext(senderId); for (uint i = 0; i < numOfMsgs; i++) { auto preProcessReplyMsg = make_shared(senderId, clientId, @@ -735,7 +618,7 @@ TEST(requestPreprocessingState_test, validatePreProcessBatchReplyMsg) { ConcordAssertEQ(preProcessBatchReplyMsg->getCid(), cid); ConcordAssertEQ(msgs.size(), numOfMsgs); uint i = 0; - SigManager::instance(sigManager[repInfo.myId()].get()); + switchReplicaContext(repInfo.myId()); for (const auto& msg : msgs) { msg->validate(repInfo); ConcordAssertEQ(msg->clientId(), clientId); @@ -748,7 +631,7 @@ TEST(requestPreprocessingState_test, validatePreProcessBatchReplyMsg) { } // Verify that requests and the batch have been successfully released in case no pre-processing consensus reached -TEST(requestPreprocessingState_test, batchCancelledNoConsensusReached) { +TEST_F(PreprocessingStateTest, batchCancelledNoConsensusReached) { const uint numOfReplicas = 4; bftEngine::impl::ReplicasInfo repInfo(replicaConfig, false, false); @@ -784,7 +667,7 @@ TEST(requestPreprocessingState_test, batchCancelledNoConsensusReached) { auto replyMsgHandlerCallback = msgHandlersRegPtr->getCallback(bftEngine::impl::MsgCode::PreProcessBatchReply); uint overallRepliesSize = 0; for (uint senderId = 1; senderId < numOfReplicas; senderId++) { - SigManager::instance(sigManager[senderId].get()); + switchReplicaContext(senderId); PreProcessReplyMsgsList repliesList; for (uint i = 0; i < numOfMsgsInBatch; i++) { auto preProcessReplyMsg = make_shared(senderId, @@ -806,18 +689,18 @@ TEST(requestPreprocessingState_test, batchCancelledNoConsensusReached) { std::make_unique(clientId, senderId, repliesList, cid, overallRepliesSize, viewNum)); } - usleep(reqTimeoutMilli * 1000); + usleep(reqTimeoutMilli * 10000); // Verify that the requests/batch have been released - ConcordAssertEQ(ongoingBatch->isBatchInProcess(), false); - ConcordAssertEQ(preProcessor.getOngoingReqIdForClient(clientId, 0), 0); - ConcordAssertEQ(preProcessor.getOngoingReqIdForClient(clientId, 1), 0); + ASSERT_EQ(ongoingBatch->isBatchInProcess(), false); + ASSERT_EQ(preProcessor.getOngoingReqIdForClient(clientId, 0), 0); + ASSERT_EQ(preProcessor.getOngoingReqIdForClient(clientId, 1), 0); clearDiagnosticsHandlers(); for (auto& req : reqBatch) delete req; replica.stop(); } -TEST(requestPreprocessingState_test, requestTimedOut) { +TEST_F(PreprocessingStateTest, requestTimedOut) { setUpConfiguration_7(); bftEngine::impl::ReplicasInfo repInfo(replicaConfig, false, false); @@ -837,7 +720,7 @@ TEST(requestPreprocessingState_test, requestTimedOut) { clearDiagnosticsHandlers(); } -TEST(requestPreprocessingState_test, primaryCrashDetected) { +TEST_F(PreprocessingStateTest, primaryCrashDetected) { setUpConfiguration_7(); bftEngine::impl::ReplicasInfo repInfo(replicaConfig, false, false); @@ -860,7 +743,7 @@ TEST(requestPreprocessingState_test, primaryCrashDetected) { clearDiagnosticsHandlers(); } -TEST(requestPreprocessingState_test, primaryCrashNotDetected) { +TEST_F(PreprocessingStateTest, primaryCrashNotDetected) { setUpConfiguration_7(); bftEngine::impl::ReplicasInfo repInfo(replicaConfig, false, false); @@ -901,7 +784,7 @@ TEST(requestPreprocessingState_test, primaryCrashNotDetected) { } // Verify that in case of timed out requests, the requests and the batch get properly released -TEST(requestPreprocessingState_test, batchMsgTimedOutOnNonPrimary) { +TEST_F(PreprocessingStateTest, batchMsgTimedOutOnNonPrimary) { setUpConfiguration_7(); bftEngine::impl::ReplicasInfo repInfo(replicaConfig, false, false); @@ -947,7 +830,7 @@ TEST(requestPreprocessingState_test, batchMsgTimedOutOnNonPrimary) { } // Verify that in case of timed out requests, the requests and the batch get properly released -TEST(requestPreprocessingState_test, batchMsgTimedOutOnPrimary) { +TEST_F(PreprocessingStateTest, batchMsgTimedOutOnPrimary) { setUpConfiguration_7(); bftEngine::impl::ReplicasInfo repInfo(replicaConfig, false, false); @@ -992,7 +875,7 @@ TEST(requestPreprocessingState_test, batchMsgTimedOutOnPrimary) { replica.stop(); } -TEST(requestPreprocessingState_test, handlePreProcessBatchRequestMsgByNonPrimary) { +TEST_F(PreprocessingStateTest, handlePreProcessBatchRequestMsgByNonPrimary) { setUpConfiguration_7(); bftEngine::impl::ReplicasInfo repInfo(replicaConfig, false, false); @@ -1040,7 +923,7 @@ TEST(requestPreprocessingState_test, handlePreProcessBatchRequestMsgByNonPrimary clearDiagnosticsHandlers(); } -TEST(requestPreprocessingState_test, rejectMsgWithInvalidView) { +TEST_F(PreprocessingStateTest, rejectMsgWithInvalidView) { bftEngine::impl::ReplicasInfo repInfo(replicaConfig, false, false); DummyReplica replica(repInfo); replica.setPrimary(false); @@ -1116,7 +999,7 @@ TEST(requestPreprocessingState_test, rejectMsgWithInvalidView) { ConcordAssert(preProcessor.validateBatchRequestMsgCorrectness(batchRequestValidView)); ConcordAssert(!preProcessor.validateBatchRequestMsgCorrectness(batchRequestInvalidView)); - SigManager::instance(sigManager[senderId].get()); + switchReplicaContext(senderId); PreProcessReplyMsgsList batch; uint overallRepliesSize = 0; for (uint i = 0; i < numOfMsgs; i++) { @@ -1141,8 +1024,8 @@ TEST(requestPreprocessingState_test, rejectMsgWithInvalidView) { make_unique(clientId, senderId, batch, cid, overallRepliesSize, oldViewNum); replica.setPrimary(true); - SigManager::instance(sigManager[repInfo.myId()].get()); - ConcordAssert(preProcessor.validateBatchReplyMsgCorrectness(batchReplyValidView)); + switchReplicaContext(repInfo.myId()); + ConcordAssert(preProcessor.validateBatchReplyMsgCorrectness(batchReplyValidView)); ConcordAssert(!preProcessor.validateBatchReplyMsgCorrectness(batchReplyInvalidView)); replica.stop(); } @@ -1153,10 +1036,6 @@ int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); logging::initLogger("logging.properties"); logging::Logger::getInstance("preprocessor_test").setLogLevel(log4cplus::ERROR_LOG_LEVEL); - if (replicaConfig.replicaMsgSigningAlgo == SignatureAlgorithm::EdDSA) { - replicaPrivKeys = replicaEdDSAPrivKeys; - replicaPubKeys = replicaEdDSAPubKeys; - } setUpConfiguration_4(); RequestProcessingState::init(numOfRequiredReplies, &preProcessorRecorder); PreProcessReplyMsg::setPreProcessorHistograms(&preProcessorRecorder); diff --git a/libs/crypto/src/threshsign/eddsa/EdDSAMultisigFactory.cpp b/libs/crypto/src/threshsign/eddsa/EdDSAMultisigFactory.cpp index 20ef486b98..11a3900770 100644 --- a/libs/crypto/src/threshsign/eddsa/EdDSAMultisigFactory.cpp +++ b/libs/crypto/src/threshsign/eddsa/EdDSAMultisigFactory.cpp @@ -31,11 +31,10 @@ IThresholdVerifier *EdDSAMultisigFactory::newVerifier(ShareID reqSigners, const std::vector &verifKeysStr) const { using SingleVerifier = EdDSAMultisigVerifier::SingleVerifier; using PublicKey = SingleVerifier::VerifierKeyType; - ConcordAssertEQ(verifKeysStr.size(), static_cast::size_type>(totalSigners + 1)); + ConcordAssertEQ(verifKeysStr.size(), static_cast::size_type>(totalSigners)); std::vector verifiers; - verifiers.emplace_back(SingleVerifier{EdDSAThreshsignPublicKey{EdDSAThreshsignPublicKey::ByteArray{}}}); std::transform( - verifKeysStr.begin() + 1, verifKeysStr.end(), std::back_inserter(verifiers), [](const std::string &publicKeyHex) { + verifKeysStr.begin(), verifKeysStr.end(), std::back_inserter(verifiers), [](const std::string &publicKeyHex) { LOG_DEBUG(EDDSA_MULTISIG_LOG, KVLOG(publicKeyHex)); return EdDSAMultisigVerifier::SingleVerifier(fromHexString(publicKeyHex)); }); @@ -54,15 +53,10 @@ IThresholdFactory::SignersVerifierTuple EdDSAMultisigFactory::newRandomSigners(N NumSharesType numSigners) const { std::vector allPrivateKeys; std::vector allVerifiers; - std::vector> signers(static_cast(numSigners + 1)); - - // One-based indices - allPrivateKeys.push_back(EdDSAThreshsignPrivateKey{EdDSAThreshsignPrivateKey::ByteArray{}}); - allVerifiers.emplace_back(EdDSAThreshsignPublicKey{EdDSAThreshsignPublicKey::ByteArray{}}); - signers[0].reset(new EdDSAMultisigSigner(allPrivateKeys[0], (uint32_t)0)); + std::vector> signers(static_cast(numSigners)); ConcordAssertLE(reqSigners, numSigners); - for (size_t i = 1; i <= static_cast(numSigners); i++) { + for (size_t i = 0; i < static_cast(numSigners); i++) { auto [privateKey, publicKey] = newKeyPair(); const auto &priv = *dynamic_cast(privateKey.get()); const auto &pub = *dynamic_cast(publicKey.get()); diff --git a/libs/crypto/threshsign/eddsa/EdDSAMultisigVerifier.h b/libs/crypto/threshsign/eddsa/EdDSAMultisigVerifier.h index 5f68990837..ee429eea6d 100644 --- a/libs/crypto/threshsign/eddsa/EdDSAMultisigVerifier.h +++ b/libs/crypto/threshsign/eddsa/EdDSAMultisigVerifier.h @@ -56,6 +56,7 @@ class EdDSAMultisigVerifier : public IThresholdVerifier { /// This is stubbed as there is no meaning to a single public key in this implementation const IPublicKey &getPublicKey() const override; const IShareVerificationKey &getShareVerificationKey(ShareID signer) const override; + const concord::crypto::IVerifier &getVerifier(size_t verifier) const; bool verifySingleSignature(const concord::Byte *msg, size_t msgLen, const SingleEdDSASignature &signature) const; ~EdDSAMultisigVerifier() override = default; From 3a0429a3e9a54569e224c93d84a0ae76966a91e2 Mon Sep 17 00:00:00 2001 From: Ynon Flum Date: Thu, 15 Dec 2022 19:45:08 +0200 Subject: [PATCH 02/18] Redirect SigManager calls from replicas to CryptoManager to support main key rotations Extracted CryptoManager.cpp from CryptoManager.hpp --- bftengine/CMakeLists.txt | 1 + bftengine/include/bftengine/CryptoManager.hpp | 163 ++++-------- bftengine/src/bftengine/CryptoManager.cpp | 231 ++++++++++++++++++ bftengine/src/bftengine/SigManager.cpp | 185 +++++++++----- bftengine/src/bftengine/SigManager.hpp | 139 +++++------ 5 files changed, 469 insertions(+), 250 deletions(-) create mode 100644 bftengine/src/bftengine/CryptoManager.cpp diff --git a/bftengine/CMakeLists.txt b/bftengine/CMakeLists.txt index f8637cc863..0c40f7e9da 100644 --- a/bftengine/CMakeLists.txt +++ b/bftengine/CMakeLists.txt @@ -38,6 +38,7 @@ set(corebft_source_files src/bftengine/ReplicaFactory.cpp src/bftengine/RequestsBatchingLogic.cpp src/bftengine/ReplicaStatusHandlers.cpp + src/bftengine/CryptoManager.cpp src/bcstatetransfer/BCStateTran.cpp src/bcstatetransfer/BCStateTranInterface.cpp src/bcstatetransfer/RVBManager.cpp diff --git a/bftengine/include/bftengine/CryptoManager.hpp b/bftengine/include/bftengine/CryptoManager.hpp index 7a4985e5dd..e2d8a67a6b 100644 --- a/bftengine/include/bftengine/CryptoManager.hpp +++ b/bftengine/include/bftengine/CryptoManager.hpp @@ -11,6 +11,7 @@ #pragma once +#include #include #include "log/logger.hpp" @@ -31,81 +32,60 @@ class CryptoManager : public IKeyExchanger, public IMultiSigKeyGenerator { * Singleton access method * For the first time should be called with a non-null argument */ - static CryptoManager& instance(std::unique_ptr&& cryptoSys = nullptr) { - static CryptoManager cm_(std::move(cryptoSys)); - return cm_; - } - std::shared_ptr thresholdSignerForSlowPathCommit(const SeqNum sn) const { - return get(sn)->thresholdSigner_; - } - std::shared_ptr thresholdVerifierForSlowPathCommit(const SeqNum sn) const { - return get(sn)->thresholdVerifierForSlowPathCommit_; - } - std::shared_ptr thresholdSignerForCommit(const SeqNum sn) const { - return get(sn)->thresholdSigner_; - } - std::shared_ptr thresholdVerifierForCommit(const SeqNum sn) const { - return get(sn)->thresholdVerifierForCommit_; - } - std::shared_ptr thresholdSignerForOptimisticCommit(const SeqNum sn) const { - return get(sn)->thresholdSigner_; - } - std::shared_ptr thresholdVerifierForOptimisticCommit(const SeqNum sn) const { - return get(sn)->thresholdVerifierForOptimisticCommit_; - } - - std::unique_ptr& getLatestCryptoSystem() const { return cryptoSystems_.rbegin()->second->cryptosys_; } + + static std::shared_ptr s_cm; + + static std::shared_ptr init(std::unique_ptr&& cryptoSys); + static CryptoManager& instance(); + static void reset(std::shared_ptr other); + + std::shared_ptr thresholdSignerForSlowPathCommit(const SeqNum sn) const; + std::shared_ptr thresholdVerifierForSlowPathCommit(const SeqNum sn) const; + std::shared_ptr thresholdSignerForCommit(const SeqNum sn) const; + std::shared_ptr thresholdVerifierForCommit(const SeqNum sn) const; + std::shared_ptr thresholdSignerForOptimisticCommit(const SeqNum sn) const; + std::shared_ptr thresholdVerifierForOptimisticCommit(const SeqNum sn) const; + std::unique_ptr& getLatestCryptoSystem() const; /** * @return An algorithm identifier for the latest threshold signature scheme */ - concord::crypto::SignatureAlgorithm getLatestSignatureAlgorithm() const { - const std::unordered_map typeToAlgorithm{ - {MULTISIG_EDDSA_SCHEME, concord::crypto::SignatureAlgorithm::EdDSA}, - }; - auto currentType = getLatestCryptoSystem()->getType(); - return typeToAlgorithm.at(currentType); - } + concord::crypto::SignatureAlgorithm getLatestSignatureAlgorithm() const; // IMultiSigKeyGenerator methods - std::tuple generateMultisigKeyPair() override { - LOG_INFO(logger(), "Generating new multisig key pair"); - auto [priv, pub] = getLatestCryptoSystem()->generateNewKeyPair(); - return {priv, pub, getLatestSignatureAlgorithm()}; - } + std::tuple generateMultisigKeyPair() override; // IKeyExchanger methods // onPrivateKeyExchange and onPublicKeyExchange callbacks for a given checkpoint may be called in a different order. - // Therefore the first called will create a CryptoSys + // Therefore the first called will create ca CryptoSys void onPrivateKeyExchange(const std::string& secretKey, const std::string& verificationKey, - const SeqNum& sn) override { - LOG_INFO(logger(), "Private key exchange:" << KVLOG(sn, verificationKey)); - auto sys_wrapper = create(sn); - sys_wrapper->cryptosys_->updateKeys(secretKey, verificationKey); - sys_wrapper->init(); - } + const SeqNum& sn) override; + void onPublicKeyExchange(const std::string& verificationKey, const std::uint16_t& signerIndex, - const SeqNum& sn) override { - LOG_INFO(logger(), "Public key exchange:" << KVLOG(sn, signerIndex, verificationKey)); - auto sys = create(sn); - // the +1 is due to Crypto system starts counting from 1 - sys->cryptosys_->updateVerificationKey(verificationKey, signerIndex + 1); - sys->init(); - } - - void onCheckpoint(const uint64_t& checkpoint) { - LOG_INFO(logger(), "Checkpoint: " << checkpoint); - // clearOldKeys(); - } + const SeqNum& sn) override; + + void onCheckpoint(uint64_t newCheckpoint); + + // Important note: + // CryptoManager's cryptosystems are currently implemented using a naive eddsa multisig scheme + // The following methods break the abstraction of the threshsign library in order + // to extract ISigner and IVerifier object. + // This abstraction is broken to allow using the consensus key as the replica's main key (In SigManager), thus + // enabling an operator to change it (key rotation). + // This code will need to be refactored if a different cryptosystem is used. + IThresholdSigner* getSigner(SeqNum seq) const; + IThresholdVerifier* getMultisigVerifier(SeqNum seq) const; + std::array, 2> getLatestVerifiers() const; + std::array, 2> getLatestSigners() const; private: /** * Holds Cryptosystem, signers and verifiers per checkpoint */ struct CryptoSystemWrapper { - CryptoSystemWrapper(std::unique_ptr&& cs) : cryptosys_(std::move(cs)) {} + CryptoSystemWrapper(std::unique_ptr&& cs); CryptoSystemWrapper(const CryptoSystemWrapper&) = delete; std::unique_ptr cryptosys_; std::shared_ptr thresholdSigner_; @@ -119,68 +99,33 @@ class CryptoManager : public IKeyExchanger, public IMultiSigKeyGenerator { // verifier of a threshold signature (for threshold N out of N) std::shared_ptr thresholdVerifierForOptimisticCommit_; - void init() { - std::uint16_t f{ReplicaConfig::instance().getfVal()}; - std::uint16_t c{ReplicaConfig::instance().getcVal()}; - std::uint16_t numSigners{ReplicaConfig::instance().getnumReplicas()}; - thresholdSigner_.reset(cryptosys_->createThresholdSigner()); - thresholdVerifierForSlowPathCommit_.reset(cryptosys_->createThresholdVerifier(f * 2 + c + 1)); - thresholdVerifierForCommit_.reset(cryptosys_->createThresholdVerifier(f * 3 + c + 1)); - thresholdVerifierForOptimisticCommit_.reset(cryptosys_->createThresholdVerifier(numSigners)); - } + void init(); }; + using SeqToSystemMap = std::map>; // accessing existing Cryptosystems - std::shared_ptr get(const SeqNum& sn) const { - // find last chckp that is less than a chckp of a given sn - uint64_t chckp = (sn - 1) / checkpointWindowSize; - auto it = cryptoSystems_.rbegin(); - while (it != cryptoSystems_.rend()) { - if (it->first <= chckp) { - // LOG_TRACE(logger(), KVLOG(sn, chckp, it->first, it->second)); - return it->second; - } - it++; - } - LOG_FATAL(logger(), "Cryptosystem not found for checkpoint: " << chckp << "seqnum: " << sn); - ConcordAssert(false && "should never reach here"); - } + std::shared_ptr get(const SeqNum& sn) const; + // create CryptoSys for sn if still doesn't exist - std::shared_ptr create(const SeqNum& sn) { - // Cryptosystem for this sn will be activated upon reaching a second checkpoint from now - uint64_t chckp = sn / checkpointWindowSize + 2; - if (auto it = cryptoSystems_.find(chckp); it != cryptoSystems_.end()) return it->second; - // copy construct new Cryptosystem from a last one as we want it to include all the existing keys - std::unique_ptr cs = - std::make_unique(*cryptoSystems_.rbegin()->second->cryptosys_.get()); - LOG_INFO(logger(), "created new Cryptosytem for checkpoint: " << chckp); - return cryptoSystems_.insert(std::make_pair(chckp, std::make_shared(std::move(cs)))) - .first->second; - } - - // store at most 2 cryptosystems - void clearOldKeys() { - while (cryptoSystems_.size() > 2) { - LOG_INFO(logger(), "delete Cryptosytem for checkpoint: " << cryptoSystems_.begin()->first); - cryptoSystems_.erase(cryptoSystems_.begin()); - } - } - - CryptoManager(std::unique_ptr&& cryptoSys) { - // default cryptosystem is always at chckp 0 - cryptoSystems_.insert(std::make_pair(0, std::make_shared(std::move(cryptoSys)))); - cryptoSystems_.begin()->second->init(); - } - logging::Logger& logger() const { - static logging::Logger logger_ = logging::getLogger("concord.bft.crypto-mgr"); - return logger_; - } + std::shared_ptr create(const SeqNum& sn); + + + CryptoManager(std::unique_ptr&& cryptoSys); + logging::Logger& logger() const; CryptoManager(const CryptoManager&) = delete; CryptoManager(const CryptoManager&&) = delete; CryptoManager& operator=(const CryptoManager&) = delete; CryptoManager& operator=(const CryptoManager&&) = delete; + void assertMapSizeValid() const; + const SeqToSystemMap& getSeqToSystem() const; + // chckp -> CryptoSys - std::map> cryptoSystems_; + // TODO: this can be converted to a concurrent queue instead of using a mutex + SeqToSystemMap cryptoSystems_; + // Old cryptosystems can be removed on a checkpoint, which might invalidate + // existing cryptoSystems_ iterators. We thus protect cryptoSystems_ access with a mutex + // and rely on shared_ptr to keep old cryptosystems alive in concurrent threads when they lag + mutable std::mutex mutex_; }; } // namespace bftEngine diff --git a/bftengine/src/bftengine/CryptoManager.cpp b/bftengine/src/bftengine/CryptoManager.cpp new file mode 100644 index 0000000000..05ae8b0375 --- /dev/null +++ b/bftengine/src/bftengine/CryptoManager.cpp @@ -0,0 +1,231 @@ +// Concord +// +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// +// This product is licensed to you under the Apache 2.0 license (the "License"). You may not use this product except in +// compliance with the Apache 2.0 License. +// +// This product may include a number of subcomponents with separate copyright notices and license terms. Your use of +// these subcomponents is subject to the terms and conditions of the sub-component's license, as noted in the LICENSE +// file. + +#include +#include "ReplicaConfig.hpp" +#include "Logger.hpp" +#include "CryptoManager.hpp" + +namespace bftEngine { + +std::shared_ptr CryptoManager::s_cm{nullptr}; + +std::shared_ptr CryptoManager::init(std::unique_ptr&& cryptoSys) { + ConcordAssert(cryptoSys != nullptr); + s_cm.reset(new CryptoManager(std::move(cryptoSys))); + LOG_INFO(s_cm->logger(), "CryptoManager Initialized"); + return s_cm; +} + +CryptoManager& CryptoManager::instance() { + ConcordAssertNE(s_cm.get(), nullptr); + return *s_cm; +} +std::shared_ptr CryptoManager::thresholdSignerForSlowPathCommit(const SeqNum sn) const { + return get(sn)->thresholdSigner_; +} +std::shared_ptr CryptoManager::thresholdVerifierForSlowPathCommit(const SeqNum sn) const { + return get(sn)->thresholdVerifierForSlowPathCommit_; +} +std::shared_ptr CryptoManager::thresholdSignerForCommit(const SeqNum sn) const { + return get(sn)->thresholdSigner_; +} +std::shared_ptr CryptoManager::thresholdVerifierForCommit(const SeqNum sn) const { + return get(sn)->thresholdVerifierForCommit_; +} +std::shared_ptr CryptoManager::thresholdSignerForOptimisticCommit(const SeqNum sn) const { + return get(sn)->thresholdSigner_; +} +std::shared_ptr CryptoManager::thresholdVerifierForOptimisticCommit(const SeqNum sn) const { + return get(sn)->thresholdVerifierForOptimisticCommit_; +} + +std::unique_ptr& CryptoManager::getLatestCryptoSystem() const { + return getSeqToSystem().rbegin()->second->cryptosys_; +} + +/** + * @return An algorithm identifier for the latest threshold signature scheme + */ +concord::crypto::SignatureAlgorithm CryptoManager::getLatestSignatureAlgorithm() const { + const std::unordered_map typeToAlgorithm{ + {MULTISIG_BLS_SCHEME, concord::crypto::SignatureAlgorithm::BLS}, + {THRESHOLD_BLS_SCHEME, concord::crypto::SignatureAlgorithm::BLS}, + {MULTISIG_EDDSA_SCHEME, concord::crypto::SignatureAlgorithm::EdDSA}, + }; + auto currentType = getLatestCryptoSystem()->getType(); + return typeToAlgorithm.at(currentType); +} + +// IMultiSigKeyGenerator methods +std::tuple CryptoManager::generateMultisigKeyPair() { + LOG_INFO(logger(), "Generating new multisig key pair"); + auto [priv, pub] = getLatestCryptoSystem()->generateNewKeyPair(); + return {priv, pub, getLatestSignatureAlgorithm()}; +} + +void CryptoManager::onPrivateKeyExchange(const std::string& secretKey, + const std::string& verificationKey, + const SeqNum& sn) { + LOG_INFO(logger(), "Private key exchange:" << KVLOG(sn, verificationKey)); + auto sys_wrapper = create(sn); + sys_wrapper->cryptosys_->updateKeys(secretKey, verificationKey); + sys_wrapper->init(); +} +void CryptoManager::onPublicKeyExchange(const std::string& verificationKey, + const std::uint16_t& signerIndex, + const SeqNum& sn) { + LOG_INFO(logger(), "Public key exchange:" << KVLOG(sn, signerIndex, verificationKey)); + auto sys = create(sn); + // the +1 is due to Crypto system starts counting from 1 + sys->cryptosys_->updateVerificationKey(verificationKey, signerIndex + 1); + sys->init(); +} + +void CryptoManager::onCheckpoint(uint64_t newCheckpoint) { + std::lock_guard guard(mutex_); + LOG_INFO(logger(), + "Checkpoint " << newCheckpoint << " reached, removing stale keys" << KVLOG(getSeqToSystem().size())); + std::vector checkpointsToRemove; + for (auto& [checkpoint, _] : getSeqToSystem()) { + if (checkpoint + 2 < newCheckpoint) { + checkpointsToRemove.push_back(checkpoint); + } + } + + if (!checkpointsToRemove.empty()) { + for (auto checkpointToRemove : checkpointsToRemove) { + LOG_INFO(logger(), + "Removing stale cryptosystem " << KVLOG(checkpointToRemove, newCheckpoint, getSeqToSystem().size())); + cryptoSystems_.erase(checkpointToRemove); + assertMapSizeValid(); + // TODO: persist key store + } + } + + // clearOldKeys(); +} + +IThresholdSigner* CryptoManager::getSigner(SeqNum seq) const { + auto signer = get(seq)->thresholdSigner_.get(); + return signer; +} + +IThresholdVerifier* CryptoManager::getMultisigVerifier(SeqNum seq) const { + auto verifier = get(seq)->thresholdVerifierForOptimisticCommit_.get(); + return verifier; +} + + +std::array, 2> CryptoManager::getLatestVerifiers() const { + std::lock_guard guard(mutex_); + auto riter = getSeqToSystem().rbegin(); + std::array, 2> result; + result[0] = riter->second->thresholdVerifierForOptimisticCommit_; + riter++; + if (riter->second != nullptr) { + result[1] = riter->second->thresholdVerifierForOptimisticCommit_; + } + return result; +} + +std::array, 2> CryptoManager::getLatestSigners() const { + std::lock_guard guard(mutex_); + auto riter = getSeqToSystem().rbegin(); + std::array, 2> result; + result[0] = riter->second->thresholdSigner_; + riter++; + if (riter->second != nullptr) { + result[1] = riter->second->thresholdSigner_; + } + return result; +} + + +CryptoManager::CryptoSystemWrapper::CryptoSystemWrapper(std::unique_ptr&& cs) + : cryptosys_(std::move(cs)) {} + +void CryptoManager::CryptoSystemWrapper::init() { + std::uint16_t f{ReplicaConfig::instance().getfVal()}; + std::uint16_t c{ReplicaConfig::instance().getcVal()}; + std::uint16_t numSigners{ReplicaConfig::instance().getnumReplicas()}; + thresholdSigner_.reset(cryptosys_->createThresholdSigner()); + thresholdVerifierForSlowPathCommit_.reset(cryptosys_->createThresholdVerifier(f * 2 + c + 1)); + thresholdVerifierForCommit_.reset(cryptosys_->createThresholdVerifier(f * 3 + c + 1)); + thresholdVerifierForOptimisticCommit_.reset(cryptosys_->createThresholdVerifier(numSigners)); +} + +std::shared_ptr CryptoManager::get(const SeqNum& sn) const { + std::lock_guard guard(mutex_); + // find last chckp that is less than a chckp of a given sn + const uint64_t checkpointUpperBound = (sn - 1) / checkpointWindowSize; + for (auto riter = getSeqToSystem().rbegin(); riter != getSeqToSystem().rend(); ++riter) { + auto& [checkpointCandidate, cryptosystem] = *riter; + if (checkpointCandidate <= checkpointUpperBound) { + LOG_INFO(logger(), + "Found cryptosystem for " << KVLOG(sn, + checkpointUpperBound, + checkpointCandidate, + cryptosystem, + getSeqToSystem().size(), + checkpointWindowSize)); + return cryptosystem; + } + } + + + LOG_FATAL( + logger(), + "Cryptosystem not found " << KVLOG(sn, checkpointUpperBound, getSeqToSystem().size(), checkpointWindowSize)); + ConcordAssert(false && "should never reach here"); +} + +std::shared_ptr CryptoManager::create(const SeqNum& sn) { + assertMapSizeValid(); + // Cryptosystem for this sn will be activated upon reaching a second checkpoint from now + uint64_t chckp = (sn / checkpointWindowSize) + 2; + if (auto it = getSeqToSystem().find(chckp); it != getSeqToSystem().end()) return it->second; + // copy construct new Cryptosystem from a last one as we want it to include all the existing keys + std::unique_ptr cs = + std::make_unique(*getSeqToSystem().rbegin()->second->cryptosys_.get()); + LOG_INFO(logger(), "created new Cryptosytem for checkpoint: " << chckp); + auto ret = + cryptoSystems_.insert(std::make_pair(chckp, std::make_shared(std::move(cs)))).first->second; + assertMapSizeValid(); + return ret; +} + +CryptoManager::CryptoManager(std::unique_ptr&& cryptoSys) + : cryptoSystems_{{0, std::make_shared(std::move(cryptoSys))}} { + // default cryptosystem is always at chckp 0 + cryptoSystems_.begin()->second->init(); +} + +logging::Logger& CryptoManager::logger() const { + static logging::Logger logger_ = logging::getLogger("concord.bft.crypto-mgr"); + return logger_; +} + +void CryptoManager::assertMapSizeValid() const { + ConcordAssertGE(cryptoSystems_.size(), 1); + ConcordAssertLE(cryptoSystems_.size(), 3); +} + +const CryptoManager::SeqToSystemMap& CryptoManager::getSeqToSystem() const { + assertMapSizeValid(); + return cryptoSystems_; +} + +void CryptoManager::reset(std::shared_ptr other) { + s_cm = other; +} + +} // namespace bftEngine diff --git a/bftengine/src/bftengine/SigManager.cpp b/bftengine/src/bftengine/SigManager.cpp index 929ade0987..826235a252 100644 --- a/bftengine/src/bftengine/SigManager.cpp +++ b/bftengine/src/bftengine/SigManager.cpp @@ -18,6 +18,9 @@ #include "ReplicaConfig.hpp" #include "util/hex_tools.hpp" #include "crypto/factory.hpp" +#include "CryptoManager.hpp" +#include "crypto/threshsign/eddsa/EdDSAMultisigVerifier.h" +#include "crypto/threshsign/eddsa/EdDSAMultisigSigner.h" using namespace std; @@ -30,6 +33,8 @@ using concord::crypto::KeyFormat; concord::messages::keys_and_signatures::ClientsPublicKeys clientsPublicKeys_; +std::shared_ptr SigManager::s_sm; + std::string SigManager::getClientsPublicKeys() { std::shared_lock lock(mutex_); std::vector output; @@ -37,15 +42,23 @@ std::string SigManager::getClientsPublicKeys() { return std::string(output.begin(), output.end()); } -SigManager* SigManager::initImpl( - ReplicaId myId, - const Key& mySigPrivateKey, - const std::set>& publicKeysOfReplicas, - KeyFormat replicasKeysFormat, - const std::set>>* publicKeysOfClients, - KeyFormat clientsKeysFormat, - const std::optional>& operatorKey, - ReplicasInfo& replicasInfo) { +SigManager* SigManager::instance() { + ConcordAssertNE(s_sm.get(), nullptr); + return s_sm.get(); +} + +void SigManager::reset(std::shared_ptr other) { + s_sm = other; +} + +std::shared_ptr SigManager::init(ReplicaId myId, + const Key& mySigPrivateKey, + const std::set>& publicKeysOfReplicas, + KeyFormat replicasKeysFormat, + const std::set>>* publicKeysOfClients, + KeyFormat clientsKeysFormat, + const std::optional>& operatorKey, + const ReplicasInfo& replicasInfo) { vector> publickeys; map publicKeysMapping; size_t lowBound, highBound; @@ -72,7 +85,7 @@ SigManager* SigManager::initImpl( if (publicKeysOfClients) { // Multiple clients might be signing with the same private key (1 to many relation) - // Also, we do not enforce to have all range between [lowBound, highBound] construcred. We might want to have less + // Also, we do not enforce to have all range between [lowBound, highBound] constructed. We might want to have less // principal ids mapped to keys than what is stated in the range. lowBound = numRoReplicas + numReplicas + numOfClientProxies; highBound = lowBound + numOfExternalClients + numOfInternalClients + numOfClientServices - 1; @@ -91,60 +104,26 @@ SigManager* SigManager::initImpl( } LOG_INFO(GL, "Done Compute Start ctor for SigManager with " << KVLOG(publickeys.size(), publicKeysMapping.size())); - return new SigManager( + auto ret = std::shared_ptr{new SigManager( myId, - numReplicas, make_pair(mySigPrivateKey, replicasKeysFormat), publickeys, publicKeysMapping, ((ReplicaConfig::instance().clientTransactionSigningEnabled) && (publicKeysOfClients != nullptr)), operatorKey, - replicasInfo); -} + replicasInfo)}; -SigManager* SigManager::init(ReplicaId myId, - const Key& mySigPrivateKey, - const std::set>& publicKeysOfReplicas, - KeyFormat replicasKeysFormat, - const std::set>>* publicKeysOfClients, - KeyFormat clientsKeysFormat, - ReplicasInfo& replicasInfo) { - return init(myId, - mySigPrivateKey, - publicKeysOfReplicas, - replicasKeysFormat, - publicKeysOfClients, - clientsKeysFormat, - std::nullopt, - replicasInfo); -} -SigManager* SigManager::init(ReplicaId myId, - const Key& mySigPrivateKey, - const std::set>& publicKeysOfReplicas, - KeyFormat replicasKeysFormat, - const std::set>>* publicKeysOfClients, - KeyFormat clientsKeysFormat, - const std::optional>& operatorKey, - ReplicasInfo& replicasInfo) { - SigManager* sm = initImpl(myId, - mySigPrivateKey, - publicKeysOfReplicas, - replicasKeysFormat, - publicKeysOfClients, - clientsKeysFormat, - operatorKey, - replicasInfo); - return SigManager::instance(sm); + reset(ret); + return ret; } SigManager::SigManager(PrincipalId myId, - uint16_t numReplicas, const pair& mySigPrivateKey, const vector>& publickeys, const map& publicKeysMapping, bool clientTransactionSigningEnabled, - const std::optional>& operatorKey, - ReplicasInfo& replicasInfo) + const std::optional>& operatorKey + const ReplicasInfo& replicasInfo) : myId_(myId), clientTransactionSigningEnabled_(clientTransactionSigningEnabled), replicasInfo_(replicasInfo), @@ -192,7 +171,7 @@ SigManager::SigManager(PrincipalId myId, } } - clientsPublicKeys_.version = 1; // version `1` suggests RSAVerifier. + clientsPublicKeys_.version = 2; // version `1` suggests RSAVerifier. LOG_DEBUG(KEY_EX_LOG, "Map contains " << clientsPublicKeys_.ids_to_keys.size() << " public clients keys"); metrics_component_.Register(); @@ -214,8 +193,7 @@ SigManager::SigManager(PrincipalId myId, } LOG_INFO(GL, - "Signature Manager initialized with own private key, " - << verifiers_.size() << " verifiers and " << publicKeyIndexToVerifier.size() << " other principals"); + "SigManager initialized: " << KVLOG(myId_, verifiers_.size(), publicKeyIndexToVerifier.size())); ConcordAssert(verifiers_.size() >= publickeys.size()); } @@ -225,7 +203,9 @@ uint16_t SigManager::getSigLength(PrincipalId pid) const { } else { std::shared_lock lock(mutex_); if (auto pos = verifiers_.find(pid); pos != verifiers_.end()) { - return pos->second->signatureLength(); + auto result = pos->second->signatureLength(); + // LOG_INFO(GL, "Sig size for id: " << pid << "is " << result); + return result; } else { LOG_ERROR(GL, "Unrecognized pid " << pid); return 0; @@ -233,11 +213,13 @@ uint16_t SigManager::getSigLength(PrincipalId pid) const { } } -bool SigManager::verifySig( +bool SigManager::verifyNonReplicaSig( PrincipalId pid, const concord::Byte* data, size_t dataLength, const concord::Byte* sig, uint16_t sigLength) const { bool result = false; { std::shared_lock lock(mutex_); + ConcordAssert(!replicasInfo_.isIdOfReplica(myId_) || !replicasInfo_.isIdOfReplica(pid)); + if (auto pos = verifiers_.find(pid); pos != verifiers_.end()) { result = pos->second->verifyBuffer(data, dataLength, sig, sigLength); } else { @@ -274,15 +256,76 @@ bool SigManager::verifySig( return result; } -size_t SigManager::sign(const concord::Byte* data, size_t dataLength, concord::Byte* outSig) const { - return mySigner_->signBuffer(data, dataLength, outSig); + +size_t SigManager::sign(SeqNum seq, const concord::Byte* data, size_t dataLength, concord::Byte* outSig) const { + ConcordAssert(replicasInfo_.isIdOfReplica(myId_)); + auto& signer = *reinterpret_cast(CryptoManager::instance().getSigner(seq)); + auto result = signer.signBuffer(data, dataLength, outSig); + LOG_INFO(GL, "Signing as replica with " << KVLOG(myId_, seq, signer.signatureLength(), result)); + return result; } -size_t SigManager::sign(const char* data, size_t dataLength, char* outSig) const { - return sign(reinterpret_cast(data), dataLength, reinterpret_cast(outSig)); +size_t SigManager::sign(SeqNum seq, const char* data, size_t dataLength, char* outSig) const { + return sign(seq, reinterpret_cast(data), dataLength, reinterpret_cast(outSig)); } -uint16_t SigManager::getMySigLength() const { return (uint16_t)mySigner_->signatureLength(); } +bool SigManager::verifyReplicaSig(PrincipalId replicaID, + const concord::Byte* data, + size_t dataLength, + const concord::Byte* sig, + uint16_t sigLength) const { + ConcordAssert(replicasInfo_.isIdOfReplica(myId_)); + ConcordAssert(replicasInfo_.isIdOfReplica(replicaID)); + for (auto multisigVerifier : CryptoManager::instance().getLatestVerifiers()) { + auto& verifier = reinterpret_cast(multisigVerifier.get())->getVerifier(replicaID); + LOG_INFO(GL, "Validating as replica with: " << KVLOG(myId_, replicaID, sigLength, verifier.signatureLength())); + printCallStack(); + if (verifier.verifyBuffer(data, dataLength, sig, sigLength)) { + LOG_INFO(GL, "Validation Successful " << KVLOG(myId_, replicaID, sigLength, verifier.signatureLength())); + return true; + } else { + LOG_INFO(GL, "Validation failed as replica with: " << KVLOG(replicaID, sigLength, verifier.signatureLength())); + } + } + return false; + +} + +// verify using the two last keys, once the last key's checkpoint is reached, the previous key is removed +bool SigManager::verifySig( + PrincipalId pid, const concord::Byte* data, size_t dataLength, const concord::Byte* sig, uint16_t sigLength) const { + if (replicasInfo_.isIdOfReplica(pid)) { + return verifyReplicaSig(pid, data, dataLength, sig, sigLength); + } + + return verifyNonReplicaSig(pid, data, dataLength, sig, sigLength); +} + +bool SigManager::verifyOwnSignature(const concord::Byte* data, + size_t dataLength, + const concord::Byte* expectedSignature) const { + std::vector sig(getMySigLength()); + for (auto multisigSigner : CryptoManager::instance().getLatestSigners()) { + auto* signer = reinterpret_cast(multisigSigner.get()); + signer->signBuffer(data, dataLength, sig.data()); + + if (std::memcmp(sig.data(), expectedSignature, getMySigLength()) == 0) { + LOG_INFO(GL, "Self-sig validation succeeded"); + return true; + } else { + LOG_INFO(GL, "Self-sig validation failed"); + } + } + + return false; +} + +uint16_t SigManager::getMySigLength() const { + if (replicasInfo_.isIdOfReplica(myId_)) { + return getLatestReplicaSigner()->signatureLength(); + } + return (uint16_t)mySigner_->signatureLength(); +} void SigManager::setClientPublicKey(const std::string& key, PrincipalId id, KeyFormat format) { LOG_INFO(KEY_EX_LOG, "client: " << id << " key: " << key << " format: " << (uint16_t)format); @@ -304,10 +347,30 @@ void SigManager::setClientPublicKey(const std::string& key, PrincipalId id, KeyF bool SigManager::hasVerifier(PrincipalId pid) { return verifiers_.find(pid) != verifiers_.end(); } concord::crypto::SignatureAlgorithm SigManager::getMainKeyAlgorithm() const { return concord::crypto::EdDSA; } -concord::crypto::ISigner& SigManager::getSigner() { return *mySigner_; } +concord::crypto::ISigner* SigManager::getLatestReplicaSigner() const { + auto latestSigners = CryptoManager::instance().getLatestSigners(); + auto ret = static_cast(latestSigners.begin()->get()); + LOG_INFO(GL, KVLOG((uint64_t)ret)); + return ret; +} + +std::string SigManager::getSelfPrivKey() const { + return getLatestReplicaSigner()->getPrivKey(); +} + const concord::crypto::IVerifier& SigManager::getVerifier(PrincipalId otherPrincipal) const { return *verifiers_.at(otherPrincipal); } +void SigManager::setReplicaLastExecutedSeq(SeqNum seq) { + ConcordAssert(replicasInfo_.isIdOfReplica(myId_)); + replicaLastExecutedSeq_ = seq; +} + +SeqNum SigManager::getReplicaLastExecutedSeq() { + ConcordAssert(replicasInfo_.isIdOfReplica(myId_)); + return replicaLastExecutedSeq_; +} + } // namespace impl } // namespace bftEngine diff --git a/bftengine/src/bftengine/SigManager.hpp b/bftengine/src/bftengine/SigManager.hpp index c9e4cb2acf..1fda28d137 100644 --- a/bftengine/src/bftengine/SigManager.hpp +++ b/bftengine/src/bftengine/SigManager.hpp @@ -1,6 +1,6 @@ // Concord // -// Copyright (c) 2018 VMware, Inc. All Rights Reserved. +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. // // This product is licensed to you under the Apache 2.0 license (the "License"). You may not use this product except in // compliance with the Apache 2.0 License. @@ -16,6 +16,7 @@ #include "crypto/crypto.hpp" #include "crypto/signer.hpp" #include "crypto/verifier.hpp" +#include "memory.hpp" #include #include @@ -37,61 +38,55 @@ class SigManager { typedef std::string Key; typedef uint16_t KeyIndex; - // It is the caller responsibility to deallocate (delete) the object - // NOTICE: sm should be != nullptr ONLY for testing purpose. In that case it will be used as a set function. - static SigManager* instance(SigManager* sm = nullptr) { - static SigManager* instance_ = nullptr; - - if (sm) { - instance_ = sm; - } - - return instance_; - } + virtual ~SigManager() = default; + static SigManager* instance(); + static void reset(std::shared_ptr other); // It is the caller responsibility to deallocate (delete) the object - static SigManager* init(ReplicaId myId, - const Key& mySigPrivateKey, - const std::set>& publicKeysOfReplicas, - concord::crypto::KeyFormat replicasKeysFormat, - const std::set>>* publicKeysOfClients, - concord::crypto::KeyFormat clientsKeysFormat, - const std::optional>& operatorKey, - ReplicasInfo& replicasInfo); - - // It is the caller responsibility to deallocate (delete) the object - static SigManager* init(ReplicaId myId, - const Key& mySigPrivateKey, - const std::set>& publicKeysOfReplicas, - concord::crypto::KeyFormat replicasKeysFormat, - const std::set>>* publicKeysOfClients, - concord::crypto::KeyFormat clientsKeysFormat, - ReplicasInfo& replicasInfo); + // This method is assumed to be called by a single thread + static std::shared_ptr init(PrincipalId myId, + const Key& mySigPrivateKey, + const std::set>& publicKeysOfReplicas, + concord::crypto::KeyFormat replicasKeysFormat, + const std::set>>* publicKeysOfClients, + concord::crypto::KeyFormat clientsKeysFormat, + const std::optional>& operatorKey, + const ReplicasInfo& replicasInfo); // returns 0 if pid is invalid - caller might consider throwing an exception uint16_t getSigLength(PrincipalId pid) const; + // returns false if actual verification failed, or if pid is invalid - bool verifySig(PrincipalId pid, - const concord::Byte* data, - size_t dataLength, - const concord::Byte* sig, - uint16_t sigLength) const; + virtual bool verifySig(PrincipalId replicaID, + const concord::Byte* data, + size_t dataLength, + const concord::Byte* sig, + uint16_t sigLength) const; template - bool verifySig(PrincipalId pid, const DataContainer& data, const SignatureContainer& sig) const { + bool verifySig(PrincipalId replicaID, const DataContainer& data, const SignatureContainer& sig) const { static_assert(sizeof(typename DataContainer::value_type) == sizeof(concord::Byte), "data elements are not byte-sized"); static_assert(sizeof(typename SignatureContainer::value_type) == sizeof(concord::Byte), "signature elements are not byte-sized"); - return verifySig(pid, - reinterpret_cast(data.data()), - data.size(), - reinterpret_cast(sig.data()), - static_cast(sig.size())); + return verifySig(replicaID, + reinterpret_cast(data.data()), + data.size(), + reinterpret_cast(sig.data()), + static_cast(sig.size())); } - size_t sign(const concord::Byte* data, size_t dataLength, concord::Byte* outSig) const; - size_t sign(const char* data, size_t dataLength, char* outSig) const; + // A replica may change (key rotation) its private key starting from a certain sequence number + // Only replicas sign using SigManager + size_t sign(SeqNum seq, const concord::Byte* data, size_t dataLength, concord::Byte* outSig) const; + size_t sign(SeqNum seq, const char* data, size_t dataLength, char* outSig) const; + + bool verifyReplicaSig(PrincipalId replicaID, + const concord::Byte* data, + size_t dataLength, + const concord::Byte* sig, + uint16_t sigLength) const; + uint16_t getMySigLength() const; bool isClientTransactionSigningEnabled() { return clientTransactionSigningEnabled_; } void SetAggregator(std::shared_ptr aggregator) { @@ -105,7 +100,7 @@ class SigManager { SigManager& operator=(SigManager&&) = delete; concord::crypto::SignatureAlgorithm getMainKeyAlgorithm() const; - concord::crypto::ISigner& getSigner(); + concord::crypto::ISigner* getLatestReplicaSigner() const; const concord::crypto::IVerifier& getVerifier(PrincipalId otherPrincipal) const; std::string getClientsPublicKeys(); @@ -113,35 +108,42 @@ class SigManager { if (!verifiers_.count(id)) return std::string(); return verifiers_.at(id)->getPubKey(); } - std::string getSelfPrivKey() const { return mySigner_->getPrivKey(); } + + // Used only by replicas + std::string getSelfPrivKey() const; + + void setReplicaLastExecutedSeq(SeqNum seq); + SeqNum getReplicaLastExecutedSeq(); + + bool verifyOwnSignature(const concord::Byte* data, + size_t dataLength, + const concord::Byte* expectedSignature) const; protected: static constexpr uint16_t updateMetricsAggregatorThresh = 1000; SigManager(PrincipalId myId, - uint16_t numReplicas, const std::pair& mySigPrivateKey, const std::vector>& publickeys, const std::map& publicKeysMapping, bool clientTransactionSigningEnabled, const std::optional>& operatorKey, - ReplicasInfo& replicasInfo); - - static SigManager* initImpl( - ReplicaId myId, - const Key& mySigPrivateKey, - const std::set>& publicKeysOfReplicas, - concord::crypto::KeyFormat replicasKeysFormat, - const std::set>>* publicKeysOfClients, - concord::crypto::KeyFormat clientsKeysFormat, - const std::optional>& operatorKey, - ReplicasInfo& replicasInfo); + const ReplicasInfo& replicasInfo); + + bool verifyNonReplicaSig(PrincipalId pid, + const concord::Byte* data, + size_t dataLength, + const concord::Byte* sig, + uint16_t sigLength) const; const PrincipalId myId_; + SeqNum replicaLastExecutedSeq_{0}; std::unique_ptr mySigner_; std::map> verifiers_; bool clientTransactionSigningEnabled_ = true; - ReplicasInfo& replicasInfo_; + const ReplicasInfo& replicasInfo_; + + static std::shared_ptr s_sm; struct Metrics { AtomicCounterHandle externalClientReqSigVerificationFailed_; @@ -156,29 +158,6 @@ class SigManager { mutable concordMetrics::Component metrics_component_; mutable Metrics metrics_; mutable std::shared_mutex mutex_; - // These methods bypass the singelton, and can be used (STRICTLY) for testing. - // Define the below flag in order to use them in your test. -#ifdef CONCORD_BFT_TESTING - public: - // It is the caller responsibility to deallocate (delete) the object - static SigManager* initInTesting( - ReplicaId myId, - const Key& mySigPrivateKey, - const std::set>& publicKeysOfReplicas, - concord::crypto::KeyFormat replicasKeysFormat, - const std::set>>* publicKeysOfClients, - concord::crypto::KeyFormat clientsKeysFormat, - ReplicasInfo& replicasInfo) { - return initImpl(myId, - mySigPrivateKey, - publicKeysOfReplicas, - replicasKeysFormat, - publicKeysOfClients, - clientsKeysFormat, - std::nullopt, - replicasInfo); - } -#endif }; } // namespace impl From 74a2ac20e123dac715f360ce806e8590f41e1ece Mon Sep 17 00:00:00 2001 From: Ynon Flum Date: Mon, 23 Jan 2023 09:53:39 +0200 Subject: [PATCH 03/18] Fix ownership of SigManager and CryptoManager in UTs Also Fixed apollo tests which break when publishing main key updates when replicas start --- Makefile | 42 ++- bftengine/CMakeLists.txt | 1 + bftengine/include/bftengine/CryptoManager.hpp | 7 +- .../include/bftengine/IStateTransfer.hpp | 2 + .../include/bftengine/KeyExchangeManager.hpp | 3 +- .../bcstatetransfer/AsyncStateTransferCRE.cpp | 65 +++- .../bcstatetransfer/AsyncStateTransferCRE.hpp | 21 +- bftengine/src/bcstatetransfer/BCStateTran.cpp | 6 +- bftengine/src/bftengine/CheckpointInfo.hpp | 2 +- bftengine/src/bftengine/ClientsManager.cpp | 2 + .../CollectorOfThresholdSignatures.hpp | 2 +- bftengine/src/bftengine/CryptoManager.cpp | 60 ++-- .../src/bftengine/DbCheckpointManager.cpp | 12 +- .../src/bftengine/KeyExchangeManager.cpp | 35 +- bftengine/src/bftengine/KeyStore.cpp | 16 +- bftengine/src/bftengine/KeyStore.h | 1 + .../src/bftengine/MsgHandlersRegistrator.hpp | 1 + bftengine/src/bftengine/ReadOnlyReplica.cpp | 16 + bftengine/src/bftengine/Reconfiguration.cpp | 2 +- .../src/bftengine/ReplicaForStateTransfer.cpp | 23 +- .../src/bftengine/ReplicaForStateTransfer.hpp | 2 + bftengine/src/bftengine/ReplicaImp.cpp | 235 +++++++------- bftengine/src/bftengine/ReplicaImp.hpp | 10 +- bftengine/src/bftengine/ReplicaLoader.cpp | 4 +- bftengine/src/bftengine/ReplicaLoader.hpp | 2 +- bftengine/src/bftengine/ReplicasInfo.cpp | 1 + bftengine/src/bftengine/ReplicasInfo.hpp | 17 +- bftengine/src/bftengine/RequestHandler.cpp | 5 + bftengine/src/bftengine/SigManager.cpp | 131 +++++--- bftengine/src/bftengine/SigManager.hpp | 55 ++-- bftengine/src/bftengine/SysConsts.hpp | 2 + .../ValidationOnlyIdentityManager.cpp | 52 +++ .../ValidationOnlyIdentityManager.hpp | 40 +++ bftengine/src/bftengine/ViewsManager.cpp | 2 +- .../src/bftengine/messages/CheckpointMsg.cpp | 27 +- .../src/bftengine/messages/CheckpointMsg.hpp | 13 + .../bftengine/messages/ClientRequestMsg.cpp | 3 + bftengine/src/bftengine/messages/MsgCode.hpp | 40 +-- .../messages/ReplicaAsksToLeaveViewMsg.cpp | 4 +- .../messages/ReplicaRestartReadyMsg.cpp | 11 +- .../messages/ReplicaRestartReadyMsg.hpp | 2 +- .../messages/ReplicasRestartReadyProofMsg.cpp | 2 +- .../src/bftengine/messages/ViewChangeMsg.cpp | 6 +- .../preprocessor/RequestProcessingState.cpp | 10 +- .../messages/PreProcessReplyMsg.cpp | 8 +- .../messages/PreProcessResultMsg.cpp | 5 +- .../src/preprocessor/tests/CMakeLists.txt | 4 +- .../ClientPreProcessRequestMsg_test.cpp | 2 +- .../messages/PreProcessReplyMsg_test.cpp | 4 +- .../messages/PreProcessResultMsg_test.cpp | 49 +-- .../preprocessor/tests/preprocessor_test.cpp | 48 +-- .../tests/SigManager/SigManager_test.cpp | 305 +++++++----------- .../clientsManager/ClientsManager_test.cpp | 16 +- .../tests/keyManager/KeyManager_test.cpp | 3 + .../tests/messages/CheckpointMsg_test.cpp | 2 +- .../tests/messages/ClientRequestMsg_test.cpp | 2 +- .../tests/messages/PrePrepareMsg_test.cpp | 2 +- .../ReplicaAsksToLeaveViewMsg_test.cpp | 2 +- .../messages/ReplicaRestartReadyMsg_test.cpp | 4 +- .../tests/messages/ViewChangeMsg_test.cpp | 4 +- bftengine/tests/messages/helper.cpp | 118 +++---- bftengine/tests/messages/helper.hpp | 35 +- .../msgsCertificate_test.cpp | 2 +- .../testSerialization/TestSerialization.cpp | 2 +- .../tests/testViewChange/testViewChange.cpp | 16 +- .../testViewChange/test_views_manager.cpp | 21 +- client/bftclient/src/bft_client.cpp | 6 +- .../reconfiguration/src/default_handlers.cpp | 1 + .../src/poll_based_state_client.cpp | 9 +- kvbc/include/categorization/blockchain.h | 2 + kvbc/include/reconfiguration_kvbc_handler.hpp | 7 +- kvbc/src/reconfiguration_kvbc_handler.cpp | 49 +-- kvbc/src/v4blockchain/v4_blockchain.cpp | 2 + .../integrity_checker.cpp | 10 +- libs/communication/StateControl.hpp | 11 +- libs/communication/src/AsyncTlsConnection.cpp | 23 +- libs/crypto/digest_holder.hpp | 1 + libs/crypto/openssl/EdDSASigner.hpp | 3 +- libs/crypto/openssl/EdDSAVerifier.hpp | 4 +- libs/crypto/signer.hpp | 2 +- .../threshsign/ThresholdSignaturesTypes.cpp | 61 ++-- .../threshsign/eddsa/EdDSAMultisigSigner.cpp | 5 +- .../eddsa/EdDSAMultisigVerifier.cpp | 23 +- .../eddsa/test/TestEdDSAMultisig.cpp | 18 +- .../threshsign/ThresholdSignaturesTypes.h | 7 +- libs/reconfiguration/cmf/concord.cmf | 1 + libs/reconfiguration/src/dispatcher.cpp | 35 +- libs/util/SerializableByteArray.hpp | 3 +- libs/util/assertUtils.hpp | 48 ++- tests/apollo/test_skvbc_commit_path.py | 2 +- tests/apollo/test_skvbc_dbsnapshot.py | 3 + tests/apollo/test_skvbc_reconfiguration.py | 2 +- tests/apollo/test_skvbc_state_transfer.py | 3 +- tests/apollo/util/bft.py | 4 +- tests/apollo/util/pyclient/bft_client.py | 5 +- tests/apollo/util/test_base.py | 1 - tests/config/test_comm_config.cpp | 2 +- .../TesterReplica/internalCommandsHandler.cpp | 96 ++++-- .../TesterReplica/internalCommandsHandler.hpp | 38 ++- tests/simpleKVBC/TesterReplica/main.cpp | 19 +- tools/KeyfileIOUtils.cpp | 18 +- 101 files changed, 1320 insertions(+), 853 deletions(-) mode change 100755 => 100644 bftengine/src/bftengine/ReadOnlyReplica.cpp create mode 100644 bftengine/src/bftengine/ValidationOnlyIdentityManager.cpp create mode 100644 bftengine/src/bftengine/ValidationOnlyIdentityManager.hpp diff --git a/Makefile b/Makefile index 75ab87d0d5..2eecd3d666 100644 --- a/Makefile +++ b/Makefile @@ -49,17 +49,34 @@ else TCP_ENABLED__:=OFF endif -CONCORD_BFT_CMAKE_BUILD_UTT?=ON -CONCORD_BFT_CMAKE_OMIT_TEST_OUTPUT?=OFF -CONCORD_BFT_CMAKE_KEEP_APOLLO_LOGS?=ON -CONCORD_BFT_CMAKE_RUN_APOLLO_TESTS?=ON -CONCORD_BFT_CMAKE_ASAN?=OFF -CONCORD_BFT_CMAKE_TSAN?=OFF -CONCORD_BFT_CMAKE_UBSAN?=OFF -CONCORD_BFT_CMAKE_HEAPTRACK?=OFF -CONCORD_BFT_CMAKE_CODECOVERAGE?=OFF -CONCORD_BFT_CMAKE_CCACHE?=ON -ENABLE_RESTART_RECOVERY_TESTS?=OFF +CONCORD_BFT_CMAKE_CXX_FLAGS_RELEASE?='-O3 -g' +CONCORD_BFT_CMAKE_USE_LOG4CPP?=ON +CONCORD_BFT_CMAKE_BUILD_UTT?=TRUE +CONCORD_BFT_CMAKE_BUILD_ROCKSDB_STORAGE?=TRUE +CONCORD_BFT_CMAKE_USE_S3_OBJECT_STORE?=TRUE +CONCORD_BFT_CMAKE_USE_OPENTRACING?=TRUE +CONCORD_BFT_CMAKE_USE_PROMETHEUS?=TRUE +CONCORD_BFT_CMAKE_USE_JAEGER?=TRUE +CONCORD_BFT_CMAKE_USE_JSON?=TRUE +CONCORD_BFT_CMAKE_USE_HTTPLIB?=TRUE +CONCORD_BFT_CMAKE_EXPORT_COMPILE_COMMANDS?=TRUE +CONCORD_BFT_CMAKE_OMIT_TEST_OUTPUT?=FALSE +CONCORD_BFT_CMAKE_KEEP_APOLLO_LOGS?=TRUE +CONCORD_BFT_CMAKE_RUN_APOLLO_TESTS?=TRUE +CONCORD_BFT_CMAKE_TRANSACTION_SIGNING_ENABLED?=TRUE +CONCORD_BFT_CMAKE_BUILD_SLOWDOWN?=FALSE +# Only useful with CONCORD_BFT_CMAKE_BUILD_TYPE:=Release +CONCORD_BFT_CMAKE_BUILD_KVBC_BENCH?=TRUE +# Only usefull with CONCORD_BFT_CMAKE_CXX_FLAGS_RELEASE=-O0 -g +CONCORD_BFT_CMAKE_ASAN?=FALSE +CONCORD_BFT_CMAKE_TSAN?=FALSE +CONCORD_BFT_CMAKE_UBSAN?=FALSE +CONCORD_BFT_CMAKE_HEAPTRACK?=FALSE +CONCORD_BFT_CMAKE_CODECOVERAGE?=FALSE +CONCORD_BFT_CMAKE_CCACHE?=TRUE +CONCORD_BFT_CMAKE_USE_FAKE_CLOCK_IN_TIME_SERVICE?=FALSE +ENABLE_RESTART_RECOVERY_TESTS?=FALSE +CONCORD_ENABLE_ALL_METRICS?=FALSE # Our CMake logic won't allow more one of these flags to be raised, so having this if/else logic makes sense ifeq (${CONCORD_BFT_CMAKE_ASAN},ON) @@ -160,12 +177,15 @@ __TIMESTAMP := $(shell date +%y-%m-%d_%H-%M-%S) $(shell mkdir -p ${BUILD_DIR}) $(shell echo ${__TIMESTAMP} > ${BUILD_DIR}/timestamp) +APOLLO_TIMESTAMP := $(shell date +%y-%m-%d_%H-%M-%S) + BASIC_RUN_PARAMS?=-it --init --rm --privileged=true \ --memory-swap -1 \ --cap-add NET_ADMIN --cap-add=SYS_PTRACE --ulimit core=-1 \ --name="${CONCORD_BFT_DOCKER_CONTAINER}" \ --workdir=${CONCORD_BFT_TARGET_SOURCE_PATH} \ --mount type=bind,source=${CURDIR},target=${CONCORD_BFT_TARGET_SOURCE_PATH}${CONCORD_BFT_CONTAINER_MOUNT_CONSISTENCY} \ + --env APOLLO_RUN_TEST_DIR=${APOLLO_TIMESTAMP} \ ${CONCORD_BFT_ADDITIONAL_RUN_PARAMS} ${CONCORD_BFT_DOCKER_IMAGE_FULL_PATH} .DEFAULT_GOAL:=build diff --git a/bftengine/CMakeLists.txt b/bftengine/CMakeLists.txt index 0c40f7e9da..f1140ad1c7 100644 --- a/bftengine/CMakeLists.txt +++ b/bftengine/CMakeLists.txt @@ -19,6 +19,7 @@ set(corebft_source_files src/bftengine/IncomingMsgsStorageImp.cpp src/bftengine/RetransmissionsManager.cpp src/bftengine/SigManager.cpp + src/bftengine/ValidationOnlyIdentityManager.cpp src/bftengine/ReplicasInfo.cpp src/bftengine/ViewChangeSafetyLogic.cpp src/bftengine/ViewsManager.cpp diff --git a/bftengine/include/bftengine/CryptoManager.hpp b/bftengine/include/bftengine/CryptoManager.hpp index e2d8a67a6b..35a13b9f92 100644 --- a/bftengine/include/bftengine/CryptoManager.hpp +++ b/bftengine/include/bftengine/CryptoManager.hpp @@ -75,9 +75,9 @@ class CryptoManager : public IKeyExchanger, public IMultiSigKeyGenerator { // This abstraction is broken to allow using the consensus key as the replica's main key (In SigManager), thus // enabling an operator to change it (key rotation). // This code will need to be refactored if a different cryptosystem is used. - IThresholdSigner* getSigner(SeqNum seq) const; - IThresholdVerifier* getMultisigVerifier(SeqNum seq) const; - std::array, 2> getLatestVerifiers() const; + std::shared_ptr getSigner(SeqNum seq) const; + std::shared_ptr getMultisigVerifier(SeqNum seq) const; + std::array>, 2> getLatestVerifiers() const; std::array, 2> getLatestSigners() const; private: @@ -109,7 +109,6 @@ class CryptoManager : public IKeyExchanger, public IMultiSigKeyGenerator { // create CryptoSys for sn if still doesn't exist std::shared_ptr create(const SeqNum& sn); - CryptoManager(std::unique_ptr&& cryptoSys); logging::Logger& logger() const; CryptoManager(const CryptoManager&) = delete; diff --git a/bftengine/include/bftengine/IStateTransfer.hpp b/bftengine/include/bftengine/IStateTransfer.hpp index 4627948b24..38f7e4767c 100644 --- a/bftengine/include/bftengine/IStateTransfer.hpp +++ b/bftengine/include/bftengine/IStateTransfer.hpp @@ -89,6 +89,8 @@ class IStateTransfer : public IReservedPages { // Accepts the checkpoint number as a parameter. // Callbacks must not throw. // Multiple callbacks can be added. + // TODO: the callbacks should be stored in a queue/priority queue and run in a + // predetermined order to prevent repeating logic such as loading main keys from reserved pages virtual void addOnTransferringCompleteCallback( const std::function &cb, StateTransferCallBacksPriorities priority = StateTransferCallBacksPriorities::DEFAULT) = 0; diff --git a/bftengine/include/bftengine/KeyExchangeManager.hpp b/bftengine/include/bftengine/KeyExchangeManager.hpp index 588badf94e..8153d694d3 100644 --- a/bftengine/include/bftengine/KeyExchangeManager.hpp +++ b/bftengine/include/bftengine/KeyExchangeManager.hpp @@ -20,6 +20,7 @@ #include "SysConsts.hpp" #include "crypto/crypto.hpp" #include +#include "util/filesystem.hpp" namespace bftEngine::impl { @@ -132,7 +133,7 @@ class KeyExchangeManager { } // save to secure store void save() { - LOG_INFO(KEY_EX_LOG, "Save key"); + LOG_INFO(KEY_EX_LOG, "Save keys to " << fs::absolute(secrets_file_)); std::stringstream ss; concord::serialize::Serializable::serialize(ss, data_); secretsMgr_->encryptFile(secrets_file_, ss.str()); diff --git a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp index ddb1cab14e..70a2e42e7c 100644 --- a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp +++ b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp @@ -21,6 +21,7 @@ #include "bftclient/bft_client.h" #include "ControlStateManager.hpp" #include "reconfiguration/ireconfiguration.hpp" +#include "CryptoManager.hpp" namespace bftEngine::bcst::asyncCRE { using namespace concord::client::reconfiguration; @@ -71,15 +72,7 @@ class Communication : public ICommunication { uint16_t repId_; }; -class InternalSigner : public concord::crypto::ISigner { - public: - size_t signBuffer(const concord::Byte* dataIn, size_t dataLen, concord::Byte* sigOutBuffer) override { - return bftEngine::impl::SigManager::instance()->sign(dataIn, dataLen, sigOutBuffer); - } - - size_t signatureLength() const override { return bftEngine::impl::SigManager::instance()->getMySigLength(); } - std::string getPrivKey() const override { return ""; } -}; +// Implement cryptomanager update handler class ScalingReplicaHandler : public IStateHandler { public: @@ -136,8 +129,42 @@ class ScalingReplicaHandler : public IStateHandler { return logger_; } }; -std::shared_ptr CreFactory::create(std::shared_ptr msgsCommunicator, - std::shared_ptr msgHandlers) { + +class MainKeyUpdateHandler : public IStateHandler { + public: + MainKeyUpdateHandler() { LOG_INFO(getLogger(), "Created StateTransfer CRE replica main key update handler"); } + bool validate(const State& state) const override { + concord::messages::ClientStateReply crep; + concord::messages::deserialize(state.data, crep); + if (std::holds_alternative(crep.response)) { + concord::messages::ReplicaMainKeyUpdate update = std::get(crep.response); + return true; + } + return false; + } + + bool execute(const State& state, WriteState&) override { + LOG_INFO(getLogger(), "execute MainKeyUpdateHandler"); + concord::messages::ClientStateReply crep; + concord::messages::deserialize(state.data, crep); + concord::messages::ReplicaMainKeyUpdate update = std::get(crep.response); + // TODO(yf): persist key file? + // TODO(yf): update key via cryptomanager + bftEngine::CryptoManager::instance().onPublicKeyExchange(update.key, update.sender_id, update.seq_num); + return true; + } + + private: + logging::Logger getLogger() { + static logging::Logger logger_(logging::getLogger("bftEngine::bcst::asyncCRE.MainKeyUpdateHandler")); + return logger_; + } +}; + +std::shared_ptr CreFactory::create( + std::shared_ptr msgsCommunicator, + std::shared_ptr msgHandlers, + std::unique_ptr transactionSigner) { bft::client::ClientConfig bftClientConf; auto& repConfig = bftEngine::ReplicaConfig::instance(); bftClientConf.f_val = repConfig.fVal; @@ -154,7 +181,7 @@ std::shared_ptr CreFactory::create(std::shared_ptr< bftClientConf.replicas_master_key_folder_path = std::nullopt; std::unique_ptr comm = std::make_unique(msgsCommunicator, msgHandlers); bft::client::Client* bftClient = new bft::client::Client(std::move(comm), bftClientConf); - bftClient->setTransactionSigner(new InternalSigner()); + bftClient->setTransactionSigner(transactionSigner.release()); Config cre_config; cre_config.id_ = repConfig.replicaId; cre_config.interval_timeout_ms_ = 1000; @@ -162,6 +189,20 @@ std::shared_ptr CreFactory::create(std::shared_ptr< auto cre = std::make_shared(cre_config, pbc, std::make_shared()); if (!bftEngine::ReplicaConfig::instance().isReadOnly) cre->registerHandler(std::make_shared()); + cre->registerHandler(std::make_shared()); return cre; } + +size_t ReplicaCRESigner::signBuffer(const concord::Byte* dataIn, size_t dataLen, concord::Byte* sigOutBuffer) const { + LOG_INFO(GL, "ReplicaCRESigner signing"); + auto* sigManager = bftEngine::impl::SigManager::instance(); + //return sigManager->getLastReplicaSigner()->signBuffer(dataIn, dataLen, sigOutBuffer); + return sigManager->sign(sigManager->getReplicaLastExecutedSeq(), dataIn, dataLen, sigOutBuffer); +} + +size_t ReplicaCRESigner::signatureLength() const { + return bftEngine::impl::SigManager::instance()->getLastReplicaSigner()->signatureLength(); +} +std::string ReplicaCRESigner::getPrivKey() const { ConcordAssert(false); } + } // namespace bftEngine::bcst::asyncCRE diff --git a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.hpp b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.hpp index c1b70c6e96..dfb6735697 100644 --- a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.hpp +++ b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.hpp @@ -1,3 +1,4 @@ + // Concord // // Copyright (c) 2021 VMware, Inc. All Rights Reserved. @@ -22,6 +23,24 @@ namespace bftEngine::bcst::asyncCRE { class CreFactory { public: static std::shared_ptr create( - std::shared_ptr msgsCommunicator, std::shared_ptr msgHandlers); + std::shared_ptr msgsCommunicator, + std::shared_ptr msgHandlers, + std::unique_ptr transactionSigner); }; + +/* + * TODO: Transaction signing needs to be disabled for replicas, as state transfer may change + * a replica's consensus (and therefore main) key. Apollo tests currently run with transaction signing enabled, + * we therefore use the latest key published by this replica. + * If a consensus was reached over a key exchange which was initiated by this replica + * prior to issuing CRE client requests, the other replicas are guaranteed to be able to use + * this replica's latest main key assuming an honest execution of the exchange. + */ +class ReplicaCRESigner : public concord::crypto::ISigner { + public: + size_t signBuffer(const concord::Byte* dataIn, size_t dataLen, concord::Byte* sigOutBuffer) const override; + size_t signatureLength() const override; + std::string getPrivKey() const override; +}; + } // namespace bftEngine::bcst::asyncCRE diff --git a/bftengine/src/bcstatetransfer/BCStateTran.cpp b/bftengine/src/bcstatetransfer/BCStateTran.cpp index fef210b02a..29e9e684a0 100644 --- a/bftengine/src/bcstatetransfer/BCStateTran.cpp +++ b/bftengine/src/bcstatetransfer/BCStateTran.cpp @@ -448,8 +448,8 @@ void BCStateTran::initImpl(uint64_t maxNumOfRequiredStoredCheckpoints, } void BCStateTran::startRunningImpl(IReplicaForStateTransfer *r) { - LOG_INFO(logger_, "Starting"); FetchingState fs = getFetchingState(); + LOG_INFO(logger_, "Starting" << KVLOG(fs)); // TODO - The next lines up to comment 'XXX' do not belong here (CRE) - move outside if (!config_.isReadOnly && cre_) { @@ -3431,9 +3431,9 @@ void BCStateTran::processData(bool lastInBatch, uint32_t rvbDigestsSize) { LOG_ERROR(logger_, "Setting RVB digests into RVB manager failed!"); badDataFromCurrentSourceReplica = true; } else { -#ifdef ENABLE_ALL_METRICS + #ifdef ENABLE_ALL_METRICS metrics_.overall_rvb_digest_groups_validated_++; -#endif + #endif } } diff --git a/bftengine/src/bftengine/CheckpointInfo.hpp b/bftengine/src/bftengine/CheckpointInfo.hpp index 22ddbad590..c2522b57be 100644 --- a/bftengine/src/bftengine/CheckpointInfo.hpp +++ b/bftengine/src/bftengine/CheckpointInfo.hpp @@ -97,7 +97,7 @@ class CheckpointInfo { const uint16_t myId = info.myId(); - const uint16_t numOfReps = info.numberOfReplicas(); + const uint16_t numOfReps = info.getNumberOfReplicas(); const uint16_t C = info.cVal(); const uint16_t F = info.fVal(); ConcordAssert(numOfReps == 3 * F + 2 * C + 1); diff --git a/bftengine/src/bftengine/ClientsManager.cpp b/bftengine/src/bftengine/ClientsManager.cpp index e1465d6955..b44f3e22b0 100644 --- a/bftengine/src/bftengine/ClientsManager.cpp +++ b/bftengine/src/bftengine/ClientsManager.cpp @@ -148,6 +148,7 @@ void ClientsManager::RepliesInfo::deleteReplyIfNeededSafe(NodeIdType clientId, if (repliesBiMap_.right.find(reqIndex) != repliesBiMap_.right.end()) { savedReplyId = repliesBiMap_.right.at(reqIndex); if (savedReplyId > reqSeqNum) { + printCallStack(); LOG_WARN(CL_MNGR, "A newer reply was already saved" << KVLOG(clientId, reqSeqNum, savedReplyId, reqIndex)); } } @@ -178,6 +179,7 @@ uint16_t ClientsManager::RepliesInfo::getIndex(ReqId reqSeqNum) const { return (replyIt != repliesBiMap_.left.end()) ? replyIt->second : 0; } + /*************************** Class ClientsManager ***************************/ // Initialize: diff --git a/bftengine/src/bftengine/CollectorOfThresholdSignatures.hpp b/bftengine/src/bftengine/CollectorOfThresholdSignatures.hpp index 064712e8c5..e95dfe8424 100644 --- a/bftengine/src/bftengine/CollectorOfThresholdSignatures.hpp +++ b/bftengine/src/bftengine/CollectorOfThresholdSignatures.hpp @@ -396,7 +396,7 @@ class CollectorOfThresholdSignatures { // if verification failed again // signer index starts with 1, therefore shareId-1 std::set replicasWithBadSigs; - for (const auto& shareId : acc->getInvalidShareIds()) replicasWithBadSigs.insert((uint16_t)shareId - 1); + for (const auto& shareId : acc->getInvalidShareIds()) replicasWithBadSigs.insert((uint16_t)shareId); // send failed message auto iMsg(ExternalFunc::createInterCombinedSigFailed(expectedSeqNumber, expectedView, replicasWithBadSigs)); repMsgsStorage->pushInternalMsg(std::move(iMsg)); diff --git a/bftengine/src/bftengine/CryptoManager.cpp b/bftengine/src/bftengine/CryptoManager.cpp index 05ae8b0375..d6f3671917 100644 --- a/bftengine/src/bftengine/CryptoManager.cpp +++ b/bftengine/src/bftengine/CryptoManager.cpp @@ -85,27 +85,32 @@ void CryptoManager::onPublicKeyExchange(const std::string& verificationKey, const SeqNum& sn) { LOG_INFO(logger(), "Public key exchange:" << KVLOG(sn, signerIndex, verificationKey)); auto sys = create(sn); - // the +1 is due to Crypto system starts counting from 1 - sys->cryptosys_->updateVerificationKey(verificationKey, signerIndex + 1); + sys->cryptosys_->updateVerificationKey(verificationKey, signerIndex); sys->init(); } void CryptoManager::onCheckpoint(uint64_t newCheckpoint) { std::lock_guard guard(mutex_); - LOG_INFO(logger(), - "Checkpoint " << newCheckpoint << " reached, removing stale keys" << KVLOG(getSeqToSystem().size())); std::vector checkpointsToRemove; for (auto& [checkpoint, _] : getSeqToSystem()) { + if (getSeqToSystem().size() - checkpointsToRemove.size() == 1) { + break; + } + if (checkpoint + 2 < newCheckpoint) { checkpointsToRemove.push_back(checkpoint); } } + LOG_INFO(logger(), + "Checkpoint " << newCheckpoint << " reached, removing " << checkpointsToRemove.size() << " stale keys" + << KVLOG(getSeqToSystem().size())); + if (!checkpointsToRemove.empty()) { for (auto checkpointToRemove : checkpointsToRemove) { - LOG_INFO(logger(), - "Removing stale cryptosystem " << KVLOG(checkpointToRemove, newCheckpoint, getSeqToSystem().size())); cryptoSystems_.erase(checkpointToRemove); + LOG_INFO(logger(), + "Removed stale cryptosystem " << KVLOG(checkpointToRemove, newCheckpoint, getSeqToSystem().size())); assertMapSizeValid(); // TODO: persist key store } @@ -114,25 +119,24 @@ void CryptoManager::onCheckpoint(uint64_t newCheckpoint) { // clearOldKeys(); } -IThresholdSigner* CryptoManager::getSigner(SeqNum seq) const { - auto signer = get(seq)->thresholdSigner_.get(); +std::shared_ptr CryptoManager::getSigner(SeqNum seq) const { + auto signer = get(seq)->thresholdSigner_; return signer; } -IThresholdVerifier* CryptoManager::getMultisigVerifier(SeqNum seq) const { - auto verifier = get(seq)->thresholdVerifierForOptimisticCommit_.get(); +std::shared_ptr CryptoManager::getMultisigVerifier(SeqNum seq) const { + auto verifier = get(seq)->thresholdVerifierForOptimisticCommit_; return verifier; } - -std::array, 2> CryptoManager::getLatestVerifiers() const { +std::array>, 2> CryptoManager::getLatestVerifiers() const { std::lock_guard guard(mutex_); auto riter = getSeqToSystem().rbegin(); - std::array, 2> result; - result[0] = riter->second->thresholdVerifierForOptimisticCommit_; + std::array>, 2> result; + result[0] = {riter->first, riter->second->thresholdVerifierForOptimisticCommit_}; riter++; - if (riter->second != nullptr) { - result[1] = riter->second->thresholdVerifierForOptimisticCommit_; + if (riter != getSeqToSystem().rend() && riter->second != nullptr) { + result[1] = {riter->first, riter->second->thresholdVerifierForOptimisticCommit_}; } return result; } @@ -142,14 +146,13 @@ std::array, 2> CryptoManager::getLatestSigners auto riter = getSeqToSystem().rbegin(); std::array, 2> result; result[0] = riter->second->thresholdSigner_; - riter++; - if (riter->second != nullptr) { + ++riter; + if (riter != getSeqToSystem().rend() && riter->second != nullptr) { result[1] = riter->second->thresholdSigner_; } return result; } - CryptoManager::CryptoSystemWrapper::CryptoSystemWrapper(std::unique_ptr&& cs) : cryptosys_(std::move(cs)) {} @@ -181,7 +184,6 @@ std::shared_ptr CryptoManager::get(const Seq } } - LOG_FATAL( logger(), "Cryptosystem not found " << KVLOG(sn, checkpointUpperBound, getSeqToSystem().size(), checkpointWindowSize)); @@ -189,6 +191,7 @@ std::shared_ptr CryptoManager::get(const Seq } std::shared_ptr CryptoManager::create(const SeqNum& sn) { + std::lock_guard guard(mutex_); assertMapSizeValid(); // Cryptosystem for this sn will be activated upon reaching a second checkpoint from now uint64_t chckp = (sn / checkpointWindowSize) + 2; @@ -196,9 +199,16 @@ std::shared_ptr CryptoManager::create(const // copy construct new Cryptosystem from a last one as we want it to include all the existing keys std::unique_ptr cs = std::make_unique(*getSeqToSystem().rbegin()->second->cryptosys_.get()); - LOG_INFO(logger(), "created new Cryptosytem for checkpoint: " << chckp); - auto ret = - cryptoSystems_.insert(std::make_pair(chckp, std::make_shared(std::move(cs)))).first->second; + + while (cryptoSystems_.size() > 2) { + auto currentSystem = cryptoSystems_.begin(); + cryptoSystems_.erase(currentSystem); + LOG_INFO(logger(), "Removed stale cryptosystem with checkpoint " << currentSystem->first); + } + + auto insert_result = cryptoSystems_.insert(std::make_pair(chckp, std::make_shared(std::move(cs)))); + auto ret = insert_result.first->second; + LOG_INFO(logger(), "created new cryptosystem for checkpoint: " << chckp << ", insertion success: " << insert_result.second); assertMapSizeValid(); return ret; } @@ -224,8 +234,6 @@ const CryptoManager::SeqToSystemMap& CryptoManager::getSeqToSystem() const { return cryptoSystems_; } -void CryptoManager::reset(std::shared_ptr other) { - s_cm = other; -} +void CryptoManager::reset(std::shared_ptr other) { s_cm = other; } } // namespace bftEngine diff --git a/bftengine/src/bftengine/DbCheckpointManager.cpp b/bftengine/src/bftengine/DbCheckpointManager.cpp index 1c3e7d1422..19ea02be10 100644 --- a/bftengine/src/bftengine/DbCheckpointManager.cpp +++ b/bftengine/src/bftengine/DbCheckpointManager.cpp @@ -341,16 +341,16 @@ void DbCheckpointManager::sendInternalCreateDbCheckpointMsg(const SeqNum& seqNum req.sender = replica_id; req.seqNum = seqNum; req.noop = noop; - std::vector data_vec; + std::vector data_vec; concord::messages::db_checkpoint_msg::serialize(data_vec, req); - std::string sig(SigManager::instance()->getMySigLength(), '\0'); - SigManager::instance()->sign(reinterpret_cast(data_vec.data()), data_vec.size(), sig.data()); - req.signature = std::vector(sig.begin(), sig.end()); + // TODO: this signature does not appear to be validated anywhere in the codebase + req.signature.resize(SigManager::instance()->getMySigLength()); + SigManager::instance()->sign(seqNum, data_vec.data(), data_vec.size(), req.signature.data()); data_vec.clear(); concord::messages::db_checkpoint_msg::serialize(data_vec, req); - std::string strMsg(data_vec.begin(), data_vec.end()); std::string cid = "replicaDbCheckpoint_" + std::to_string(seqNum) + "_" + std::to_string(replica_id); - if (client_) client_->sendRequest(bftEngine::DB_CHECKPOINT_FLAG, strMsg.size(), strMsg.c_str(), cid); + if (client_) + client_->sendRequest(bftEngine::DB_CHECKPOINT_FLAG, data_vec.size(), reinterpret_cast(data_vec.data()), cid); } void DbCheckpointManager::setNextStableSeqNumToCreateSnapshot(const std::optional& seqNum) { diff --git a/bftengine/src/bftengine/KeyExchangeManager.cpp b/bftengine/src/bftengine/KeyExchangeManager.cpp index fcf9fc5802..00ade0268c 100644 --- a/bftengine/src/bftengine/KeyExchangeManager.cpp +++ b/bftengine/src/bftengine/KeyExchangeManager.cpp @@ -114,12 +114,17 @@ std::string KeyExchangeManager::onKeyExchange(const KeyExchangeMsg& kemsg, if (ReplicaConfig::instance().getkeyExchangeOnStart() && (publicKeys_.numOfExchangedReplicas() <= liveQuorumSize)) LOG_INFO(KEY_EX_LOG, "Exchanged [" << publicKeys_.numOfExchangedReplicas() << "] out of [" << liveQuorumSize << "]" - << KVLOG(kemsg.repID)); + << KVLOG(kemsg.repID, + initial_exchange_, + exchanged(), + ReplicaConfig::instance().publishReplicasMasterKeyOnStartup)); if (!initial_exchange_ && exchanged()) { initial_exchange_ = true; - if (ReplicaConfig::instance().getkeyExchangeOnStart() && - ReplicaConfig::instance().publishReplicasMasterKeyOnStartup) + if (ReplicaConfig::instance().getkeyExchangeOnStart() + /*&&ReplicaConfig::instance().publishReplicasMasterKeyOnStartup*/ + ) { sendMainPublicKey(); + } } return "ok"; } @@ -149,7 +154,7 @@ void KeyExchangeManager::loadPublicKeys() { if (ReplicaConfig::instance().getkeyExchangeOnStart() && exchanged()) { ConcordAssertGE(num_loaded, liveQuorumSize); } - LOG_INFO(KEY_EX_LOG, "building crypto system after state transfer"); + LOG_INFO(KEY_EX_LOG, "building crypto system after state transfer" << KVLOG(num_loaded)); notifyRegistry(); } @@ -201,7 +206,10 @@ void KeyExchangeManager::exchangeTlsKeys(const std::string& type, const SeqNum& std::vector data_vec; concord::messages::serialize(data_vec, req); std::string sig(SigManager::instance()->getMySigLength(), '\0'); - SigManager::instance()->sign(reinterpret_cast(data_vec.data()), data_vec.size(), sig.data()); + SigManager::instance()->sign(SigManager::instance()->getReplicaLastExecutedSeq(), + reinterpret_cast(data_vec.data()), + data_vec.size(), + sig.data()); req.signature = std::vector(sig.begin(), sig.end()); data_vec.clear(); concord::messages::serialize(data_vec, req); @@ -214,22 +222,28 @@ void KeyExchangeManager::exchangeTlsKeys(const SeqNum& bft_sn) { exchangeTlsKeys("server", bft_sn); exchangeTlsKeys("client", bft_sn); metrics_->tls_key_exchange_requests_++; - bft::communication::StateControl::instance().restartComm(repID_); LOG_INFO(KEY_EX_LOG, "Replica has generated a new tls keys"); + bft::communication::StateControl::instance().restartComm(repID_); + LOG_INFO(KEY_EX_LOG, "Replica communication restarted after tls exchange"); } void KeyExchangeManager::sendMainPublicKey() { + auto [seq, latestPublicKey] = SigManager::instance()->getMyLatestPublicKey(); concord::messages::ReconfigurationRequest req; req.sender = repID_; req.command = concord::messages::ReplicaMainKeyUpdate{repID_, - SigManager::instance()->getPublicKeyOfVerifier(repID_), + latestPublicKey, "hex", - static_cast(SigManager::instance()->getMainKeyAlgorithm())}; + static_cast(SigManager::instance()->getMainKeyAlgorithm()), + static_cast(seq)}; // Mark this request as an internal one std::vector data_vec; concord::messages::serialize(data_vec, req); std::string sig(SigManager::instance()->getMySigLength(), '\0'); - SigManager::instance()->sign(reinterpret_cast(data_vec.data()), data_vec.size(), sig.data()); + SigManager::instance()->sign(SigManager::instance()->getReplicaLastExecutedSeq(), + reinterpret_cast(data_vec.data()), + data_vec.size(), + sig.data()); req.signature = std::vector(sig.begin(), sig.end()); data_vec.clear(); concord::messages::serialize(data_vec, req); @@ -276,6 +290,7 @@ void KeyExchangeManager::generateConsensusKeyAndSendInternalClientMsg(const SeqN msg.algorithm = candidate.algorithm; LOG_INFO(KEY_EX_LOG, "Sending consensus key exchange :" << KVLOG(candidate.cid, msg.pubkey, msg.algorithm)); client_->sendRequest(bftEngine::KEY_EXCHANGE_FLAG, msg, candidate.cid); + sendMainPublicKey(); metrics_->sent_key_exchange_counter++; } @@ -430,7 +445,7 @@ bool KeyExchangeManager::PrivateKeys::load() { LOG_INFO( KEY_EX_LOG, "loaded generated private key for: " << KVLOG(data_.generated.sn, data_.generated.cid, data_.generated.pub)); - for (const auto& it : data_.keys) LOG_INFO(KEY_EX_LOG, "loaded private key for sn: " << it.first); + for (const auto& it : data_.keys) LOG_INFO(KEY_EX_LOG, "loaded private key for sn: " << it.first << KVLOG(it.second)); return true; } diff --git a/bftengine/src/bftengine/KeyStore.cpp b/bftengine/src/bftengine/KeyStore.cpp index 6d2c40177a..7a088da772 100644 --- a/bftengine/src/bftengine/KeyStore.cpp +++ b/bftengine/src/bftengine/KeyStore.cpp @@ -30,6 +30,7 @@ uint16_t ClusterKeyStore::loadAllReplicasKeyStoresFromReservedPages() { std::optional ClusterKeyStore::loadReplicaKeyStoreFromReserevedPages( const uint16_t& repID) { + LOG_INFO(KEY_EX_LOG, "Loading replica keystore from reserved pages" << KVLOG(repID)); if (!loadReservedPage(repID, buffer_.size(), buffer_.data())) { LOG_INFO(KEY_EX_LOG, "Failed to load reserved page for replica " << repID << ", first start?"); return {}; @@ -38,7 +39,19 @@ std::optional ClusterKeyStore::loadReplicaKeyStoreF std::istringstream iss(buffer_); PublicKeys ks; PublicKeys::deserialize(iss, ks); - for (auto [sn, pk] : ks.keys) LOG_DEBUG(KEY_EX_LOG, "rid: " << repID << " seqnum: " << sn << " pubkey: " << pk); + //std::vector keysToRemove; + for (auto [sn, pk] : ks.keys) { + /*if (ks.keys.size() - keysToRemove.size() > 2) { + keysToRemove.push_back(sn); + }*/ + LOG_INFO(KEY_EX_LOG, "Deserialized public key from reserved pages: " << KVLOG(repID, sn, pk)); + } + + /*for (auto sn : keysToRemove) { + LOG_INFO(KEY_EX_LOG, "Removing stale key from keystore" << KVLOG(sn, ks.keys[sn])); + ks.remove(sn); + }*/ + return ks; } catch (const std::exception& e) { LOG_FATAL(KEY_EX_LOG, @@ -48,6 +61,7 @@ std::optional ClusterKeyStore::loadReplicaKeyStoreF } void ClusterKeyStore::saveReplicaKeyStoreToReserevedPages(const uint16_t& repID) { + LOG_INFO(KEY_EX_LOG, "Saving keystore to reserved pages" << KVLOG(repID)); PublicKeys clusterKey; try { clusterKey = clusterKeys_.at(repID); diff --git a/bftengine/src/bftengine/KeyStore.h b/bftengine/src/bftengine/KeyStore.h index b8d8fcbe14..a188f9b01c 100644 --- a/bftengine/src/bftengine/KeyStore.h +++ b/bftengine/src/bftengine/KeyStore.h @@ -35,6 +35,7 @@ class ClusterKeyStore : public ResPagesClient { auto res = keys.insert(std::make_pair(sn, pub)); if (!res.second) ConcordAssert(pub == res.first->second) // if existed expect same key } + void remove(const SeqNum sn) { ConcordAssertEQ(keys.erase(sn), 1);} void serializeDataMembers(std::ostream& outStream) const { serialize(outStream, keys); } void deserializeDataMembers(std::istream& inStream) { deserialize(inStream, keys); } std::map keys; diff --git a/bftengine/src/bftengine/MsgHandlersRegistrator.hpp b/bftengine/src/bftengine/MsgHandlersRegistrator.hpp index 6845a58699..517e1fa89f 100644 --- a/bftengine/src/bftengine/MsgHandlersRegistrator.hpp +++ b/bftengine/src/bftengine/MsgHandlersRegistrator.hpp @@ -49,6 +49,7 @@ using InternalMsgHandlerCallback = CallbackTypeWithRefArg; class MsgHandlersRegistrator { public: void registerMsgHandler(uint16_t msgId, const MsgHandlerCallback& callbackFunc) { + LOG_INFO(GL, "Registered message handler with code: " << msgId); msgHandlers_[msgId] = callbackFunc; } diff --git a/bftengine/src/bftengine/ReadOnlyReplica.cpp b/bftengine/src/bftengine/ReadOnlyReplica.cpp old mode 100755 new mode 100644 index f57fb068f6..1e36e441c5 --- a/bftengine/src/bftengine/ReadOnlyReplica.cpp +++ b/bftengine/src/bftengine/ReadOnlyReplica.cpp @@ -31,6 +31,7 @@ #include "util/json_output.hpp" #include "SharedTypes.hpp" #include "communication/StateControl.hpp" +#include "CryptoManager.hpp" using concordUtil::Timers; @@ -59,6 +60,12 @@ ReadOnlyReplica::ReadOnlyReplica(const ReplicaConfig &config, msgHandlers_->registerMsgHandler( MsgCode::StateTransfer, std::bind(&ReadOnlyReplica::messageHandler, this, std::placeholders::_1)); + msgHandlers_->registerMsgHandler( + MsgCode::StateTransfer, + std::bind(&ReadOnlyReplica::messageHandler, this, std::placeholders::_1)); + msgHandlers_->registerMsgHandler( + MsgCode::StateTransfer, + std::bind(&ReadOnlyReplica::messageHandler, this, std::placeholders::_1)); metrics_.Register(); SigManager::init(config_.replicaId, @@ -71,6 +78,15 @@ ReadOnlyReplica::ReadOnlyReplica(const ReplicaConfig &config, ReplicaConfig::instance().getOperatorPublicKey(), concord::crypto::KeyFormat::PemFormat}}, *repsInfo); + std::vector replicaHexPublicKeys(config_.publicKeysOfReplicas.size()); + for (auto &[i, hexKey] : config_.publicKeysOfReplicas) { + replicaHexPublicKeys[i] = hexKey; + } + auto system = std::make_unique(MULTISIG_EDDSA_SCHEME, "", replicaHexPublicKeys.size(), config.fVal); + system->loadKeys("Stub", replicaHexPublicKeys); + // The private key is also assumed to be in KeyFormat::HexaDecimalStrippedFormat + system->loadPrivateKey(config_.replicaId, config_.replicaPrivateKey); + CryptoManager::init(std::move(system)); // Register status handler for Read-Only replica registerStatusHandlers(); diff --git a/bftengine/src/bftengine/Reconfiguration.cpp b/bftengine/src/bftengine/Reconfiguration.cpp index 0af8c53888..420523751d 100644 --- a/bftengine/src/bftengine/Reconfiguration.cpp +++ b/bftengine/src/bftengine/Reconfiguration.cpp @@ -33,7 +33,7 @@ bool ReconfigurationHandler::handle(const WedgeCommand& cmd, uint32_t, const std::optional&, concord::messages::ReconfigurationResponse&) { - LOG_INFO(getLogger(), "Wedge command instructs replica to stop at sequence number " << bft_seq_num); + LOG_INFO(getLogger(), "Wedge command instructs replica to stop at next checkpoint after sequence number " << bft_seq_num); bftEngine::ControlStateManager::instance().setStopAtNextCheckpoint(bft_seq_num); if (cmd.noop == false) addCreateDbSnapshotCbOnWedge(true); diff --git a/bftengine/src/bftengine/ReplicaForStateTransfer.cpp b/bftengine/src/bftengine/ReplicaForStateTransfer.cpp index 956bb75281..0d1dbfe229 100644 --- a/bftengine/src/bftengine/ReplicaForStateTransfer.cpp +++ b/bftengine/src/bftengine/ReplicaForStateTransfer.cpp @@ -24,6 +24,8 @@ #include "KeyStore.h" #include "bcstatetransfer/AsyncStateTransferCRE.hpp" #include "client/reconfiguration/poll_based_state_client.hpp" +#include "KeyExchangeManager.hpp" +#include "SigManager.hpp" namespace bftEngine::impl { using namespace std::chrono_literals; @@ -70,12 +72,25 @@ ReplicaForStateTransfer::ReplicaForStateTransfer(const ReplicaConfig &config, } void ReplicaForStateTransfer::start() { - cre_ = bftEngine::bcst::asyncCRE::CreFactory::create(msgsCommunicator_, msgHandlers_); + /* TODO: Transaction signing needs to be disabled for replicas, as state transfer may change + * the a replica's consensus (and therefore main) key. Apollo tests currently run with transaction signing enabled, + * we therefore use the latest key published by this replica. + * If a key exchange consensus was reached by this replica, the other replicas are guaranteed to be able to use + * its latest key upon honest execution of the exchange */ + cre_ = bftEngine::bcst::asyncCRE::CreFactory::create( + msgsCommunicator_, msgHandlers_, std::make_unique()); stateTransfer->setReconfigurationEngine(cre_); stateTransfer->addOnTransferringCompleteCallback( - [this](std::uint64_t) { + [this](std::uint64_t sequence_number) { // TODO - The next lines up to comment 'YYY' do not belong here (CRE) - consider refactor or move outside if (!config_.isReadOnly) { + // Load the public keys of the other replicas from reserved pages + // so that their responses can be validated + KeyExchangeManager::instance().loadPublicKeys(); + // Make sure to sign the reconfiguration client messages using the key + // other replicas expect + SigManager::instance()->setReplicaLastExecutedSeq(sequence_number * checkpointWindowSize); + // At this point, we, if are not going to have another blocks in state transfer. So, we can safely stop CRE. // if there is a reconfiguration state change that prevents us from starting another state transfer (i.e. // scaling) then CRE probably won't work as well. @@ -165,4 +180,8 @@ Timers::Handle ReplicaForStateTransfer::addOneShotTimer(uint32_t timeoutMilli) { [this](concordUtil::Timers::Handle h) { stateTransfer->onTimer(); }); } +void ReplicaForStateTransfer::resumeCRE() { + cre_->resume(); +} + } // namespace bftEngine::impl diff --git a/bftengine/src/bftengine/ReplicaForStateTransfer.hpp b/bftengine/src/bftengine/ReplicaForStateTransfer.hpp index db0ecae4fb..d5fae54f60 100644 --- a/bftengine/src/bftengine/ReplicaForStateTransfer.hpp +++ b/bftengine/src/bftengine/ReplicaForStateTransfer.hpp @@ -69,6 +69,8 @@ class ReplicaForStateTransfer : public IReplicaForStateTransfer, public ReplicaB void onMessage(std::unique_ptr); protected: + virtual void resumeCRE(); + std::unique_ptr stateTransfer; Timers::Handle stateTranTimer_; concordMetrics::CounterHandle metric_received_state_transfers_; diff --git a/bftengine/src/bftengine/ReplicaImp.cpp b/bftengine/src/bftengine/ReplicaImp.cpp index 09260ebecd..30187273b3 100644 --- a/bftengine/src/bftengine/ReplicaImp.cpp +++ b/bftengine/src/bftengine/ReplicaImp.cpp @@ -271,14 +271,38 @@ bool ReplicaImp::validateMessage(MessageBase *msg) { DebugStatistics::onReceivedExMessage(msg->type()); } try { - msg->validate(*repsInfo); + if constexpr (std::is_same_v) { + msg->validate(*repsInfo, false); + } + else { + msg->validate(*repsInfo); + } return true; } catch (std::exception &e) { onReportAboutInvalidMessage(msg, e.what()); return false; } + +} + +/** + * validateMessage This is synchronous validate message. + * + * @param msg : Message that can validate itself as quick as possible.. + * @return : returns true if message validation succeeds else return false. + */ +/*bool ReplicaImp::validateMessage(MessageBase *msg) { + auto result = validateMessageGeneric(msg, [msg, this](){msg->validate(*repsInfo);}); + LOG_INFO(GL, "Validated generic message" << KVLOG(result)); + return result; } +bool ReplicaImp::validateMessage(CheckpointMsg *msg) { + auto result = validateMessageGeneric(msg, [msg, this](){msg->validate(*repsInfo, false);}); + LOG_INFO(GL, "Validated checkpoint message" << KVLOG(result)); + return result; +}*/ + /** * asyncValidateMessage This is a family of asynchronous message which just schedules * the validation in a thread bag and returns peacefully. This will also translate the message @@ -297,15 +321,14 @@ void ReplicaImp::asyncValidateMessage(std::unique_ptr msg) { threadPool.async( [this](auto *unValidatedMsg, auto *replicaInfo, auto *incomingMessageQueue) { - try { - unValidatedMsg->validate(*replicaInfo); + UNUSED(this); + if (!validateMessage(unValidatedMsg)) { + delete unValidatedMsg; + return; + } + CarrierMesssage *validatedCarrierMsg = new ValidatedMessageCarrierInternalMsg(unValidatedMsg); incomingMessageQueue->pushInternalMsg(validatedCarrierMsg); - } catch (std::exception &e) { - onReportAboutInvalidMessage(unValidatedMsg, e.what()); - delete unValidatedMsg; - return; - } }, msg.release(), repsInfo, @@ -378,7 +401,6 @@ void ReplicaImp::sendAndIncrementMetric(MessageBase *m, NodeIdType id, CounterHa void ReplicaImp::onReportAboutInvalidMessage(MessageBase *msg, const char *reason) { LOG_WARN(CNSUS, "Received invalid message. " << KVLOG(msg->senderId(), msg->type(), reason)); - // TODO(GG): logic that deals with invalid messages (e.g., a node that sends invalid messages may have a problem // (old version,bug,malicious,...)). } @@ -420,7 +442,10 @@ void ReplicaImp::onMessage(std::unique_ptr m if (!KeyExchangeManager::instance().exchanged() || (!KeyExchangeManager::instance().clientKeysPublished() && repsInfo->isIdOfClientProxy(senderId))) { if (!(flags & KEY_EXCHANGE_FLAG) && !(flags & CLIENTS_PUB_KEYS_FLAG)) { - LOG_INFO(KEY_EX_LOG, "Didn't complete yet, dropping msg"); + LOG_INFO(KEY_EX_LOG, + "Didn't complete yet, dropping msg" << KVLOG(KeyExchangeManager::instance().exchanged(), + KeyExchangeManager::instance().clientKeysPublished(), + repsInfo->isIdOfClientProxy(senderId))); return; } } @@ -595,7 +620,9 @@ bool ReplicaImp::checkSendPrePrepareMsgPrerequisites() { LOG_INFO(GL, "Will not send PrePrepare since next sequence number [" << primaryLastUsedSeqNum + numOfTransientPreprepareMsgs_ + 1 << "] exceeds concurrency threshold [" - << lastExecutedSeqNum + config_.getconcurrencyLevel() + activeExecutions_ << "]"); + << lastExecutedSeqNum + config_.getconcurrencyLevel() + activeExecutions_ << "]" << + KVLOG(primaryLastUsedSeqNum, numOfTransientPreprepareMsgs_, lastExecutedSeqNum, + config_.getconcurrencyLevel(), activeExecutions_)); return false; } metric_concurrency_level_.Get().Set(primaryLastUsedSeqNum + numOfTransientPreprepareMsgs_ + 1 - lastExecutedSeqNum); @@ -1894,7 +1921,12 @@ void ReplicaImp::onPrepareCombinedSigFailed(SeqNum seqNumber, return; } if ((!currentViewIsActive()) || (getCurrentView() != view) || (!mainLog->insideActiveWindow(seqNumber))) { - LOG_WARN(CNSUS, "Dropping irrelevant signature." << KVLOG(seqNumber, view)); + std::stringstream stream; + stream << "replicasWithBadSigs: "; + for (auto replicaId : replicasWithBadSigs) { + stream << replicaId << ","; + } + LOG_WARN(CNSUS, "Dropping irrelevant signature." << KVLOG(seqNumber, view) << stream.str()); return; } @@ -2625,7 +2657,7 @@ void ReplicaImp::onMessage(std::unique_ptr m const SeqNum msgLastStable = msg->getLastStableSeqNum(); const ViewNum msgViewNum = msg->getViewNumber(); - LOG_DEBUG(CNSUS, KVLOG(msgSenderId, msgLastStable, msgViewNum, lastStableSeqNum)); + LOG_DEBUG(CNSUS, "Got replica status" << KVLOG(msgSenderId, msgLastStable, msgViewNum, lastStableSeqNum)); ///////////////////////////////////////////////////////////////////////// // Checkpoints @@ -2809,6 +2841,10 @@ void ReplicaImp::onMessage(std::unique_ptr m } } + /*if (msgLastStable > lastStableSeqNum + (checkpointWindowSize * 2) && !isCollectingState()) { + startCollectingState("On receiving ReplicaStatusMsg"); + }*/ + delete msg; } @@ -3316,7 +3352,7 @@ void ReplicaImp::onTransferringCompleteImp(uint64_t newStateCheckpoint) { } return; } - lastExecutedSeqNum = newCheckpointSeqNum; + setLastExecutedSeqNum(newCheckpointSeqNum); if (ps_) { ps_->setLastExecutedSeqNum(lastExecutedSeqNum); } @@ -3445,7 +3481,7 @@ void ReplicaImp::onSeqNumIsStable(SeqNum newStableSeqNum, bool hasStateInformati if (hasStateInformation) { if (lastStableSeqNum > lastExecutedSeqNum) { - lastExecutedSeqNum = lastStableSeqNum; + setLastExecutedSeqNum(lastStableSeqNum); metric_last_executed_seq_num_.Get().Set(lastExecutedSeqNum); if (config_.getdebugStatisticsEnabled()) { DebugStatistics::onLastExecutedSequenceNumberChanged(lastExecutedSeqNum); @@ -3572,7 +3608,6 @@ void ReplicaImp::tryToSendReqMissingDataMsg(SeqNum seqNumber, bool slowPathOnly, seqNumInfo.setTimeOfLastInfoRequest(curTime); LOG_INFO(GL, "Try to request missing data. " << KVLOG(seqNumber, getCurrentView())); - ReqMissingDataMsg reqData(config_.getreplicaId(), getCurrentView(), seqNumber); const bool routerForPartialProofs = repsInfo->isCollectorForPartialProofs(getCurrentView(), seqNumber); @@ -4043,7 +4078,7 @@ ReplicaImp::ReplicaImp(const LoadedReplicaData &ld, metric_primary_last_used_seq_num_.Get().Set(primaryLastUsedSeqNum); lastStableSeqNum = ld.lastStableSeqNum; metric_last_stable_seq_num_.Get().Set(lastStableSeqNum); - lastExecutedSeqNum = ld.lastExecutedSeqNum; + setLastExecutedSeqNum(ld.lastExecutedSeqNum); metric_last_executed_seq_num_.Get().Set(lastExecutedSeqNum); strictLowerBoundOfSeqNums = ld.strictLowerBoundOfSeqNums; maxSeqNumTransferredFromPrevViews = ld.maxSeqNumTransferredFromPrevViews; @@ -4275,7 +4310,7 @@ ReplicaImp::ReplicaImp(bool firstTime, const ReplicaConfig &config, shared_ptr requestsHandler, IStateTransfer *stateTrans, - SigManager *sigManager, + std::shared_ptr sigManager, ReplicasInfo *replicasInfo, ViewsManager *viewsMgr, shared_ptr msgsCommunicator, @@ -4427,20 +4462,20 @@ ReplicaImp::ReplicaImp(bool firstTime, if (firstTime) { repsInfo = new ReplicasInfo(config_, dynamicCollectorForPartialProofs, dynamicCollectorForExecutionProofs); - sigManager_.reset(SigManager::init(config_.replicaId, - config_.replicaPrivateKey, - config_.publicKeysOfReplicas, - concord::crypto::KeyFormat::HexaDecimalStrippedFormat, - ReplicaConfig::instance().getPublicKeysOfClients(), - concord::crypto::KeyFormat::PemFormat, - {{repsInfo->getIdOfOperator(), - ReplicaConfig::instance().getOperatorPublicKey(), - concord::crypto::KeyFormat::PemFormat}}, - *repsInfo)); + sigManager_ = SigManager::init(config_.replicaId, + config_.replicaPrivateKey, + config_.publicKeysOfReplicas, + concord::crypto::KeyFormat::HexaDecimalStrippedFormat, + ReplicaConfig::instance().getPublicKeysOfClients(), + concord::crypto::KeyFormat::PemFormat, + {{repsInfo->getIdOfOperator(), + ReplicaConfig::instance().getOperatorPublicKey(), + concord::crypto::KeyFormat::PemFormat}} + *repsInfo); viewsManager = new ViewsManager(repsInfo); } else { repsInfo = replicasInfo; - sigManager_.reset(sigManager); + sigManager_ = sigManager; viewsManager = viewsMgr; } bft::communication::StateControl::instance().setGetPeerPubKeyMethod( @@ -4650,7 +4685,9 @@ void ReplicaImp::start() { KeyExchangeManager::instance().sendInitialKey(this); } else { // If key exchange is disabled, first publish the replica's main (rsa/eddsa) key to clients - if (ReplicaConfig::instance().publishReplicasMasterKeyOnStartup) KeyExchangeManager::instance().sendMainPublicKey(); + if (ReplicaConfig::instance().publishReplicasMasterKeyOnStartup) { + KeyExchangeManager::instance().sendMainPublicKey(); + } } KeyExchangeManager::instance().sendInitialClientsKeys(SigManager::instance()->getClientsPublicKeys()); } @@ -5067,6 +5104,8 @@ void ReplicaImp::executeRequests(PrePrepareMsg *ppMsg, Bitmap &requestSet, Times reqIdx++; ClientRequestMsg req((ClientRequestMsgHeader *)requestBody); + LOG_INFO(GL, "Iterating request" << KVLOG(req.senderId(), req.requestSeqNum())); + if (!requestSet.get(tmp) || req.requestLength() == 0) { InternalMessage im = RemovePendingForExecutionRequest{req.clientProxyId(), req.requestSeqNum()}; getIncomingMsgsStorage().pushInternalMsg(std::move(im)); @@ -5215,7 +5254,7 @@ void ReplicaImp::finalizeExecution() { ps_->setLastExecutedSeqNum(lastExecutedSeqNum + 1); } - lastExecutedSeqNum = lastExecutedSeqNum + 1; + setLastExecutedSeqNum(lastExecutedSeqNum + 1); if (config_.getdebugStatisticsEnabled()) { DebugStatistics::onLastExecutedSequenceNumberChanged(lastExecutedSeqNum); @@ -5334,34 +5373,7 @@ void ReplicaImp::onExecutionFinish() { handleDeferredRequests(); auto seqNumToStopAt = ControlStateManager::instance().getCheckpointToStopAt(); if (seqNumToStopAt.value_or(0) == lastExecutedSeqNum) ControlStateManager::instance().wedge(); - if (seqNumToStopAt.has_value() && seqNumToStopAt.value() > lastExecutedSeqNum && isCurrentPrimary()) { - // If after execution, we discover that we need to wedge at some future point, push a noop command to the incoming - // messages queue. - LOG_INFO(GL, "sending noop command to bring the system into wedge checkpoint"); - concord::messages::ReconfigurationRequest req; - req.sender = config_.replicaId; - req.command = concord::messages::WedgeCommand{config_.replicaId, true}; - // Mark this request as an internal one - std::vector data_vec; - concord::messages::serialize(data_vec, req); - req.signature.resize(SigManager::instance()->getMySigLength()); - SigManager::instance()->sign(data_vec.data(), data_vec.size(), req.signature.data()); - data_vec.clear(); - concord::messages::serialize(data_vec, req); - std::string strMsg(data_vec.begin(), data_vec.end()); - auto requestSeqNum = - std::chrono::duration_cast(getMonotonicTime().time_since_epoch()).count(); - auto crm = new ClientRequestMsg(internalBFTClient_->getClientId(), - RECONFIG_FLAG, - requestSeqNum, - strMsg.size(), - strMsg.c_str(), - 60000, - "wedge-noop-command-" + std::to_string(lastExecutedSeqNum + 1)); - // Now, try to send a new PrePrepare message immediately, without waiting to a new batch - onMessage(std::unique_ptr(crm)); - tryToSendPrePrepareMsg(false); - } + primaryPushNoOpIfWedgePending(lastExecutedSeqNum + 1); if (ControlStateManager::instance().getCheckpointToStopAt().has_value() && lastExecutedSeqNum == ControlStateManager::instance().getCheckpointToStopAt()) { @@ -5516,7 +5528,7 @@ void ReplicaImp::executeRequestsInPrePrepareMsg(concordUtils::SpanWrapper &paren ps_->setLastExecutedSeqNum(lastExecutedSeqNum + 1); } - lastExecutedSeqNum = lastExecutedSeqNum + 1; + setLastExecutedSeqNum(lastExecutedSeqNum + 1); if (config_.getdebugStatisticsEnabled()) { DebugStatistics::onLastExecutedSequenceNumberChanged(lastExecutedSeqNum); @@ -5712,6 +5724,7 @@ void ReplicaImp::sendResponses(PrePrepareMsg *ppMsg, IRequestsHandler::Execution req.outReplicaSpecificInfoSize, executionResult); if (replyMsg) { + LOG_INFO(GL, "Sending reply for req " << req.requestSequenceNum); send(replyMsg.get(), req.clientId); free(req.outReply); } else { @@ -5808,66 +5821,9 @@ void ReplicaImp::executeNextCommittedRequests(concordUtils::SpanWrapper &parent_ auto seqNumToStopAt = ControlStateManager::instance().getCheckpointToStopAt(); if (seqNumToStopAt.value_or(0) == lastExecutedSeqNum) ControlStateManager::instance().wedge(); - if (seqNumToStopAt.has_value() && seqNumToStopAt.value() > lastExecutedSeqNum && isCurrentPrimary()) { - // If after execution, we discover that we need to wedge at some futuer point, push a noop command to the incoming - // messages queue. - LOG_INFO(GL, "sending noop command to bring the system into wedge checkpoint"); - concord::messages::ReconfigurationRequest req; - req.sender = config_.replicaId; - req.command = concord::messages::WedgeCommand{config_.replicaId, true}; - // Mark this request as an internal one - std::vector data_vec; - concord::messages::serialize(data_vec, req); - std::string sig(SigManager::instance()->getMySigLength(), '\0'); - SigManager::instance()->sign(reinterpret_cast(data_vec.data()), data_vec.size(), sig.data()); - req.signature = std::vector(sig.begin(), sig.end()); - data_vec.clear(); - concord::messages::serialize(data_vec, req); - std::string strMsg(data_vec.begin(), data_vec.end()); - auto requestSeqNum = - std::chrono::duration_cast(getMonotonicTime().time_since_epoch()).count(); - auto crm = new ClientRequestMsg(internalBFTClient_->getClientId(), - RECONFIG_FLAG, - requestSeqNum, - strMsg.size(), - strMsg.c_str(), - 60000, - "wedge-noop-command-" + std::to_string(lastExecutedSeqNum + 1)); - // Now, try to send a new PrePrepare message immediately, without waiting to a new batch - onMessage(std::unique_ptr(crm)); - tryToSendPrePrepareMsg(false); - } - } - auto seqNumToStopAt = ControlStateManager::instance().getCheckpointToStopAt(); - if (seqNumToStopAt.has_value() && seqNumToStopAt.value() > lastExecutedSeqNum && isCurrentPrimary()) { - // If after execution, we discover that we need to wedge at some future point, push a noop command to the incoming - // messages queue. - LOG_INFO(GL, "sending noop command to bring the system into wedge checkpoint"); - concord::messages::ReconfigurationRequest req; - req.sender = config_.replicaId; - req.command = concord::messages::WedgeCommand{config_.replicaId, true}; - // Mark this request as an internal one - std::vector data_vec; - concord::messages::serialize(data_vec, req); - std::string sig(SigManager::instance()->getMySigLength(), '\0'); - SigManager::instance()->sign(reinterpret_cast(data_vec.data()), data_vec.size(), sig.data()); - req.signature = std::vector(sig.begin(), sig.end()); - data_vec.clear(); - concord::messages::serialize(data_vec, req); - std::string strMsg(data_vec.begin(), data_vec.end()); - auto current_time = - std::chrono::duration_cast(getMonotonicTime().time_since_epoch()).count(); - auto crm = new ClientRequestMsg(internalBFTClient_->getClientId(), - RECONFIG_FLAG, - current_time, - strMsg.size(), - strMsg.c_str(), - 60000, - "wedge-noop-command-" + std::to_string(lastExecutedSeqNum)); - // Now, try to send a new PrePrepare message immediately, without waiting to a new batch - onMessage(std::unique_ptr(crm)); - tryToSendPrePrepareMsg(false); + primaryPushNoOpIfWedgePending(lastExecutedSeqNum + 1); } + primaryPushNoOpIfWedgePending(lastExecutedSeqNum); if (ControlStateManager::instance().getCheckpointToStopAt().has_value() && lastExecutedSeqNum == ControlStateManager::instance().getCheckpointToStopAt()) { @@ -5907,6 +5863,47 @@ void ReplicaImp::updateCommitMetrics(const CommitPath &commitPath) { LOG_ERROR(CNSUS, "Invalid commit path value: " << static_cast(commitPath)); } } +void ReplicaImp::setLastExecutedSeqNum(SeqNum seq) { + lastExecutedSeqNum = seq; + SigManager::instance()->setReplicaLastExecutedSeq(seq); +} +void ReplicaImp::primaryPushNoOpIfWedgePending(SeqNum seq) { + auto seqNumToStopAt = ControlStateManager::instance().getCheckpointToStopAt(); + if (seqNumToStopAt.has_value() && seqNumToStopAt.value() > lastExecutedSeqNum && isCurrentPrimary()) { + // If after execution, we discover that we need to wedge at some futuer point, push a noop command to the incoming + // messages queue. + LOG_INFO(GL, "sending noop command to bring the system into wedge checkpoint"); + concord::messages::ReconfigurationRequest req; + req.sender = config_.replicaId; + req.command = concord::messages::WedgeCommand{config_.replicaId, true}; + // Mark this request as an internal one + //auto siglength = 64; + std::vector data_vec; + concord::messages::serialize(data_vec, req); + LOG_INFO(GL, "wat-1" << KVLOG(data_vec.size(), SigManager::instance()->getMySigLength(), req.signature.size())); + req.signature.resize(SigManager::instance()->getMySigLength()); + LOG_INFO(GL, "wat0" << KVLOG(data_vec.size(), SigManager::instance()->getMySigLength(), req.signature.size())); + SigManager::instance()->sign( + SigManager::instance()->getReplicaLastExecutedSeq(), data_vec.data(), data_vec.size(), req.signature.data()); + + data_vec.clear(); + concord::messages::serialize(data_vec, req); + LOG_INFO(GL, "wat1" << KVLOG(data_vec.size(), req.signature.size())); + auto requestSeqNum = + std::chrono::duration_cast(getMonotonicTime().time_since_epoch()).count(); + auto crm = new ClientRequestMsg(internalBFTClient_->getClientId(), + RECONFIG_FLAG, + requestSeqNum, + data_vec.size(), + reinterpret_cast(data_vec.data()), + 60000, + "wedge-noop-command-" + std::to_string(seq)); + LOG_INFO(GL, "wat2" << KVLOG(data_vec.size(), req.signature.size())); + // Now, try to send a new PrePrepare message immediately, without waiting to a new batch + onMessage(std::make_unique(crm)); + tryToSendPrePrepareMsg(false); + } +} // TODO(GG): the timer for state transfer !!!! diff --git a/bftengine/src/bftengine/ReplicaImp.hpp b/bftengine/src/bftengine/ReplicaImp.hpp index 5f443b9232..825e4d7374 100644 --- a/bftengine/src/bftengine/ReplicaImp.hpp +++ b/bftengine/src/bftengine/ReplicaImp.hpp @@ -94,7 +94,7 @@ class ReplicaImp : public InternalReplicaApi, public ReplicaForStateTransfer { ControllerBase* controller = nullptr; // digital signatures - std::unique_ptr sigManager_; + std::shared_ptr sigManager_; // view change logic ViewsManager* viewsManager = nullptr; @@ -371,7 +371,9 @@ class ReplicaImp : public InternalReplicaApi, public ReplicaForStateTransfer { void recoverRequests(); - bool validateMessage(MessageBase* msg); + template + bool validateMessage(MessageType *msg); + std::function getMessageValidator(); // InternalReplicaApi @@ -437,7 +439,7 @@ class ReplicaImp : public InternalReplicaApi, public ReplicaForStateTransfer { const ReplicaConfig&, shared_ptr, IStateTransfer*, - SigManager*, + std::shared_ptr, ReplicasInfo*, ViewsManager*, shared_ptr, @@ -626,6 +628,8 @@ class ReplicaImp : public InternalReplicaApi, public ReplicaForStateTransfer { // replica is stopped concord::util::CallbackRegistry<> stopCallbacks_; + void primaryPushNoOpIfWedgePending(SeqNum seq); + void setLastExecutedSeqNum(SeqNum seq); void addTimers(); void startConsensusProcess(PrePrepareMsgShPtr& pp, bool isCreatedEarlier); void startConsensusProcess(PrePrepareMsgShPtr& pp); diff --git a/bftengine/src/bftengine/ReplicaLoader.cpp b/bftengine/src/bftengine/ReplicaLoader.cpp index 8a72fe3681..8e60e23319 100644 --- a/bftengine/src/bftengine/ReplicaLoader.cpp +++ b/bftengine/src/bftengine/ReplicaLoader.cpp @@ -65,8 +65,8 @@ ReplicaLoader::ErrorCode loadConfig(LoadedReplicaData &ld) { ld.repConfig.numReplicas, ld.repConfig.numReplicas); cryptoSys->loadKeys(ld.repConfig.thresholdPublicKey_, ld.repConfig.thresholdVerificationKeys_); - cryptoSys->loadPrivateKey(ld.repConfig.replicaId + 1, ld.repConfig.thresholdPrivateKey_); - bftEngine::CryptoManager::instance(std::move(cryptoSys)); + cryptoSys->loadPrivateKey(ld.repConfig.replicaId, ld.repConfig.thresholdPrivateKey_); + bftEngine::CryptoManager::init(std::move(cryptoSys)); return Succ; } diff --git a/bftengine/src/bftengine/ReplicaLoader.hpp b/bftengine/src/bftengine/ReplicaLoader.hpp index f386e52a0b..5a5354a351 100644 --- a/bftengine/src/bftengine/ReplicaLoader.hpp +++ b/bftengine/src/bftengine/ReplicaLoader.hpp @@ -36,7 +36,7 @@ class CheckpointMsg; struct LoadedReplicaData { LoadedReplicaData() : repConfig(ReplicaConfig::instance()) {} ReplicaConfig &repConfig; - SigManager *sigManager = nullptr; + std::shared_ptr sigManager = nullptr; ReplicasInfo *repsInfo = nullptr; ViewsManager *viewsManager = nullptr; SeqNum primaryLastUsedSeqNum = 0; diff --git a/bftengine/src/bftengine/ReplicasInfo.cpp b/bftengine/src/bftengine/ReplicasInfo.cpp index b9f961a23b..3d1bb7dc75 100644 --- a/bftengine/src/bftengine/ReplicasInfo.cpp +++ b/bftengine/src/bftengine/ReplicasInfo.cpp @@ -55,6 +55,7 @@ ReplicasInfo::ReplicasInfo(const ReplicaConfig& config, bool dynamicCollectorForPartialProofs, bool dynamicCollectorForExecutionProofs) : _myId{config.replicaId}, + _isRoReplica{(_myId >= config.numReplicas) && (_myId < config.numReplicas + config.numRoReplicas)}, _numberOfReplicas{config.numReplicas}, _numberOfRoReplicas{config.numRoReplicas}, _numOfClientProxies{config.numOfClientProxies}, diff --git a/bftengine/src/bftengine/ReplicasInfo.hpp b/bftengine/src/bftengine/ReplicasInfo.hpp index f012a82990..b6a65efeaa 100644 --- a/bftengine/src/bftengine/ReplicasInfo.hpp +++ b/bftengine/src/bftengine/ReplicasInfo.hpp @@ -26,10 +26,10 @@ class ReplicasInfo { ReplicasInfo() {} ReplicaId myId() const { return _myId; } - int16_t numberOfReplicas() const { return _numberOfReplicas; } int16_t fVal() const { return _fVal; } int16_t cVal() const { return _cVal; } + bool isRoReplica() const { return _isRoReplica; } bool isIdOfReplica(NodeIdType id) const { return (id < _numberOfReplicas); } bool isIdOfPeerReplica(NodeIdType id) const { return (id < _numberOfReplicas) && (id != _myId); } bool isIdOfPeerRoReplica(NodeIdType id) const { return _idsOfPeerROReplicas.find(id) != _idsOfPeerROReplicas.end(); } @@ -40,7 +40,7 @@ class ReplicasInfo { bool isIdOfInternalClient(PrincipalId id) const { return _idsOfInternalClients.find(id) != _idsOfInternalClients.end(); } - bool isIdOfClientService(NodeIdType id) { return _idsOfClientServices.find(id) != _idsOfClientServices.end(); } + bool isIdOfClientService(NodeIdType id) const { return _idsOfClientServices.find(id) != _idsOfClientServices.end(); } bool isValidPrincipalId(PrincipalId id) const { return id <= _maxValidPrincipalId; } const std::set& idsOfPeerReplicas() const { return _idsOfPeerReplicas; } const std::set& idsOfPeerROReplicas() const { return _idsOfPeerROReplicas; } @@ -79,15 +79,16 @@ class ReplicasInfo { return getExecutionCollectors(v, n, nullptr, nullptr); } - uint16_t getNumberOfReplicas() { return _numberOfReplicas; } - uint16_t getNumberOfRoReplicas() { return _numberOfRoReplicas; } - uint16_t getNumOfClientProxies() { return _numOfClientProxies; } - uint16_t getNumberOfExternalClients() { return _numberOfExternalClients; } - uint16_t getNumberOfInternalClients() { return _numberOfInternalClients; } - uint16_t getNumberOfClientServices() { return _numberOfClientServices; } + uint16_t getNumberOfReplicas() const { return _numberOfReplicas; } + uint16_t getNumberOfRoReplicas() const { return _numberOfRoReplicas; } + uint16_t getNumOfClientProxies() const { return _numOfClientProxies; } + uint16_t getNumberOfExternalClients() const { return _numberOfExternalClients; } + uint16_t getNumberOfInternalClients() const { return _numberOfInternalClients; } + uint16_t getNumberOfClientServices() const { return _numberOfClientServices; } protected: const ReplicaId _myId = 0; + bool _isRoReplica = false; const uint16_t _numberOfReplicas = 0; const uint16_t _numberOfRoReplicas = 0; const uint16_t _numOfClientProxies = 0; diff --git a/bftengine/src/bftengine/RequestHandler.cpp b/bftengine/src/bftengine/RequestHandler.cpp index e40d9109b1..fa47543c3b 100644 --- a/bftengine/src/bftengine/RequestHandler.cpp +++ b/bftengine/src/bftengine/RequestHandler.cpp @@ -56,7 +56,11 @@ void RequestHandler::execute(IRequestsHandler::ExecutionRequestsQueue& requests, concordUtils::SpanWrapper& parent_span) { bool has_pruning_request = false; for (auto& req : requests) { + LOG_INFO(GL, "Executing request: " << + KVLOG(req.clientId, req.cid, req.flags, req.requestSequenceNum, + req.signature.size(), req.executionSequenceNum, req.requestSize)); if (req.flags & KEY_EXCHANGE_FLAG) { + //TODO(yf): maybe add the mainkeyupdate block here KeyExchangeMsg ke = KeyExchangeMsg::deserializeMsg(req.request, req.requestSize); LOG_INFO(KEY_EX_LOG, "BFT handler received KEY_EXCHANGE msg " << ke.toString()); auto resp = impl::KeyExchangeManager::instance().onKeyExchange(ke, req.executionSequenceNum, req.cid); @@ -72,6 +76,7 @@ void RequestHandler::execute(IRequestsHandler::ExecutionRequestsQueue& requests, } else if (req.flags & MsgFlag::RECONFIG_FLAG) { ReconfigurationRequest rreq; deserialize(std::vector(req.request, req.request + req.requestSize), rreq); + LOG_INFO(GL, "Executing Reconfig request: " << KVLOG(rreq.signature.size(), rreq.command.index(), rreq.sender, rreq.id)); has_pruning_request = std::holds_alternative(rreq.command); ReconfigurationResponse rsi_res = reconfig_dispatcher_.dispatch(rreq, req.executionSequenceNum, timestamp); // in case of read request return only a success part of and replica specific info in the response diff --git a/bftengine/src/bftengine/SigManager.cpp b/bftengine/src/bftengine/SigManager.cpp index 826235a252..1a5c08cf2d 100644 --- a/bftengine/src/bftengine/SigManager.cpp +++ b/bftengine/src/bftengine/SigManager.cpp @@ -47,9 +47,7 @@ SigManager* SigManager::instance() { return s_sm.get(); } -void SigManager::reset(std::shared_ptr other) { - s_sm = other; -} +void SigManager::reset(std::shared_ptr other) { s_sm = other; } std::shared_ptr SigManager::init(ReplicaId myId, const Key& mySigPrivateKey, @@ -89,15 +87,15 @@ std::shared_ptr SigManager::init(ReplicaId myId, // principal ids mapped to keys than what is stated in the range. lowBound = numRoReplicas + numReplicas + numOfClientProxies; highBound = lowBound + numOfExternalClients + numOfInternalClients + numOfClientServices - 1; - for (const auto& p : (*publicKeysOfClients)) { - ConcordAssert(!p.first.empty()); - publickeys.push_back(make_pair(p.first, clientsKeysFormat)); - for (const auto e : p.second) { - if ((e < lowBound) || (e > highBound)) { - LOG_FATAL(GL, "Invalid participant id " << KVLOG(e, lowBound, highBound)); + for (const auto& [publicKey, idSet] : (*publicKeysOfClients)) { + ConcordAssert(!publicKey.empty()); + publickeys.push_back(make_pair(publicKey, clientsKeysFormat)); + for (const auto participantId : idSet) { + if ((participantId < lowBound) || (participantId > highBound)) { + LOG_FATAL(GL, "Invalid participant id " << KVLOG(participantId, lowBound, highBound)); std::terminate(); } - publicKeysMapping.insert({e, i}); + publicKeysMapping.insert({participantId, i}); } ++i; } @@ -192,8 +190,7 @@ SigManager::SigManager(PrincipalId myId, ++i; } - LOG_INFO(GL, - "SigManager initialized: " << KVLOG(myId_, verifiers_.size(), publicKeyIndexToVerifier.size())); + LOG_INFO(GL, "SigManager initialized: " << KVLOG(myId_, verifiers_.size(), publicKeyIndexToVerifier.size())); ConcordAssert(verifiers_.size() >= publickeys.size()); } @@ -215,6 +212,7 @@ uint16_t SigManager::getSigLength(PrincipalId pid) const { bool SigManager::verifyNonReplicaSig( PrincipalId pid, const concord::Byte* data, size_t dataLength, const concord::Byte* sig, uint16_t sigLength) const { + LOG_INFO(GL, "Validating NON-replica with: " << KVLOG(myId_, pid, sigLength)); bool result = false; { std::shared_lock lock(mutex_); @@ -253,15 +251,18 @@ bool SigManager::verifyNonReplicaSig( metrics_component_.UpdateAggregator(); } } + LOG_INFO(GL, + "NON-replica validation result: " << KVLOG( + result, myId_, pid, sigLength, idOfReadOnlyReplica, idOfExternalClient)); return result; } - size_t SigManager::sign(SeqNum seq, const concord::Byte* data, size_t dataLength, concord::Byte* outSig) const { - ConcordAssert(replicasInfo_.isIdOfReplica(myId_)); - auto& signer = *reinterpret_cast(CryptoManager::instance().getSigner(seq)); - auto result = signer.signBuffer(data, dataLength, outSig); - LOG_INFO(GL, "Signing as replica with " << KVLOG(myId_, seq, signer.signatureLength(), result)); + ConcordAssert(replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica()); + auto signer = CryptoManager::instance().getSigner(seq); + auto& rawSigner = *reinterpret_cast(signer.get()); + auto result = rawSigner.signBuffer(data, dataLength, outSig); + LOG_INFO(GL, "Signing as replica with " << KVLOG(myId_, seq, rawSigner.getPrivKey(), rawSigner.signatureLength(), result)); return result; } @@ -270,25 +271,35 @@ size_t SigManager::sign(SeqNum seq, const char* data, size_t dataLength, char* o } bool SigManager::verifyReplicaSig(PrincipalId replicaID, - const concord::Byte* data, - size_t dataLength, - const concord::Byte* sig, - uint16_t sigLength) const { - ConcordAssert(replicasInfo_.isIdOfReplica(myId_)); + const concord::Byte* data, + size_t dataLength, + const concord::Byte* sig, + uint16_t sigLength) const { + LOG_INFO(GL, KVLOG(myId_)); + ConcordAssert(replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica()); ConcordAssert(replicasInfo_.isIdOfReplica(replicaID)); - for (auto multisigVerifier : CryptoManager::instance().getLatestVerifiers()) { + int i = 0; + for (auto [seq, multisigVerifier] : CryptoManager::instance().getLatestVerifiers()) { + UNUSED(seq); + if (multisigVerifier.get() == nullptr) { + continue; + } auto& verifier = reinterpret_cast(multisigVerifier.get())->getVerifier(replicaID); - LOG_INFO(GL, "Validating as replica with: " << KVLOG(myId_, replicaID, sigLength, verifier.signatureLength())); - printCallStack(); + LOG_INFO(GL, + "Validating as replica with: " << KVLOG( + myId_, replicaID, i, verifier.getPubKey(), sigLength, verifier.signatureLength())); if (verifier.verifyBuffer(data, dataLength, sig, sigLength)) { - LOG_INFO(GL, "Validation Successful " << KVLOG(myId_, replicaID, sigLength, verifier.signatureLength())); + LOG_INFO(GL, "Validation Successful " << KVLOG(myId_, replicaID, i, sigLength, verifier.signatureLength())); return true; } else { - LOG_INFO(GL, "Validation failed as replica with: " << KVLOG(replicaID, sigLength, verifier.signatureLength())); + LOG_INFO( + GL, + "Validation failed as replica with: " << KVLOG(myId_, replicaID, i, sigLength, verifier.signatureLength())); } + i++; } + LOG_WARN(GL, "Validation failed with all cryptosystems" << KVLOG(myId_, replicaID, i)); return false; - } // verify using the two last keys, once the last key's checkpoint is reached, the previous key is removed @@ -302,8 +313,8 @@ bool SigManager::verifySig( } bool SigManager::verifyOwnSignature(const concord::Byte* data, - size_t dataLength, - const concord::Byte* expectedSignature) const { + size_t dataLength, + const concord::Byte* expectedSignature) const { std::vector sig(getMySigLength()); for (auto multisigSigner : CryptoManager::instance().getLatestSigners()) { auto* signer = reinterpret_cast(multisigSigner.get()); @@ -321,8 +332,8 @@ bool SigManager::verifyOwnSignature(const concord::Byte* data, } uint16_t SigManager::getMySigLength() const { - if (replicasInfo_.isIdOfReplica(myId_)) { - return getLatestReplicaSigner()->signatureLength(); + if (replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica()) { + return getCurrentReplicaSigner()->signatureLength(); } return (uint16_t)mySigner_->signatureLength(); } @@ -347,15 +358,55 @@ void SigManager::setClientPublicKey(const std::string& key, PrincipalId id, KeyF bool SigManager::hasVerifier(PrincipalId pid) { return verifiers_.find(pid) != verifiers_.end(); } concord::crypto::SignatureAlgorithm SigManager::getMainKeyAlgorithm() const { return concord::crypto::EdDSA; } -concord::crypto::ISigner* SigManager::getLatestReplicaSigner() const { - auto latestSigners = CryptoManager::instance().getLatestSigners(); - auto ret = static_cast(latestSigners.begin()->get()); - LOG_INFO(GL, KVLOG((uint64_t)ret)); + +std::shared_ptr SigManager::getCurrentReplicaSigner() const { + ConcordAssert(replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica()); + std::shared_ptr currentSigner = + std::dynamic_pointer_cast(CryptoManager::instance().getSigner(getReplicaLastExecutedSeq())); + std::shared_ptr ret = currentSigner; + LOG_INFO(GL, KVLOG((uint64_t)ret.get())); return ret; } -std::string SigManager::getSelfPrivKey() const { - return getLatestReplicaSigner()->getPrivKey(); +std::shared_ptr SigManager::getLastReplicaSigner() const { + ConcordAssert(replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica()); + std::shared_ptr latestSigner = + std::dynamic_pointer_cast(CryptoManager::instance().getLatestSigners()[0]); + std::shared_ptr ret = latestSigner; + LOG_INFO(GL, KVLOG((uint64_t)ret.get())); + return ret; +} + +std::pair SigManager::getMyLatestPublicKey() const { + ConcordAssert(replicasInfo_.isIdOfReplica(myId_)); + auto [seq, latestVerifier] = CryptoManager::instance().getLatestVerifiers()[0]; + auto retVerifier = static_cast(latestVerifier.get()); + LOG_INFO(GL, KVLOG((uint64_t)retVerifier)); + return {seq, retVerifier->getVerifier(myId_).getPubKey()}; +} + +std::string SigManager::getSelfPrivKey() const { return getCurrentReplicaSigner()->getPrivKey(); } + +std::array SigManager::getPublicKeyOfVerifier(uint32_t id) const { + std::array publicKeys; + + if (replicasInfo_.isIdOfReplica(id)) { + int i = 0; + for (auto [seq, multisigVerifier] : CryptoManager::instance().getLatestVerifiers()) { + UNUSED(seq); + if (multisigVerifier.get() == nullptr) { + continue; + } + auto& verifier = reinterpret_cast(multisigVerifier.get())->getVerifier(id); + publicKeys[i] = verifier.getPubKey(); + ++i; + } + } + else if (verifiers_.count(id)) { + publicKeys[0] = verifiers_.at(id)->getPubKey(); + } + + return publicKeys; } const concord::crypto::IVerifier& SigManager::getVerifier(PrincipalId otherPrincipal) const { @@ -367,8 +418,8 @@ void SigManager::setReplicaLastExecutedSeq(SeqNum seq) { replicaLastExecutedSeq_ = seq; } -SeqNum SigManager::getReplicaLastExecutedSeq() { - ConcordAssert(replicasInfo_.isIdOfReplica(myId_)); +SeqNum SigManager::getReplicaLastExecutedSeq() const { + ConcordAssert(replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica()); return replicaLastExecutedSeq_; } diff --git a/bftengine/src/bftengine/SigManager.hpp b/bftengine/src/bftengine/SigManager.hpp index 1fda28d137..0787174fbe 100644 --- a/bftengine/src/bftengine/SigManager.hpp +++ b/bftengine/src/bftengine/SigManager.hpp @@ -1,3 +1,4 @@ + // Concord // // Copyright (c) 2022 VMware, Inc. All Rights Reserved. @@ -16,8 +17,8 @@ #include "crypto/crypto.hpp" #include "crypto/signer.hpp" #include "crypto/verifier.hpp" -#include "memory.hpp" - +#include "util/memory.hpp" +#include "SysConsts.hpp" #include #include #include @@ -35,8 +36,8 @@ class ReplicasInfo; class SigManager { public: - typedef std::string Key; - typedef uint16_t KeyIndex; + using Key = std::string; + using KeyIndex = uint16_t; virtual ~SigManager() = default; static SigManager* instance(); @@ -44,14 +45,15 @@ class SigManager { // It is the caller responsibility to deallocate (delete) the object // This method is assumed to be called by a single thread - static std::shared_ptr init(PrincipalId myId, - const Key& mySigPrivateKey, - const std::set>& publicKeysOfReplicas, - concord::crypto::KeyFormat replicasKeysFormat, - const std::set>>* publicKeysOfClients, - concord::crypto::KeyFormat clientsKeysFormat, - const std::optional>& operatorKey, - const ReplicasInfo& replicasInfo); + static std::shared_ptr init( + PrincipalId myId, + const Key& mySigPrivateKey, + const std::set>& publicKeysOfReplicas, + concord::crypto::KeyFormat replicasKeysFormat, + const std::set>>* publicKeysOfClients, + concord::crypto::KeyFormat clientsKeysFormat, + const std::optional>& operatorKey, + const ReplicasInfo& replicasInfo); // returns 0 if pid is invalid - caller might consider throwing an exception uint16_t getSigLength(PrincipalId pid) const; @@ -70,10 +72,10 @@ class SigManager { static_assert(sizeof(typename SignatureContainer::value_type) == sizeof(concord::Byte), "signature elements are not byte-sized"); return verifySig(replicaID, - reinterpret_cast(data.data()), - data.size(), - reinterpret_cast(sig.data()), - static_cast(sig.size())); + reinterpret_cast(data.data()), + data.size(), + reinterpret_cast(sig.data()), + static_cast(sig.size())); } // A replica may change (key rotation) its private key starting from a certain sequence number @@ -100,24 +102,26 @@ class SigManager { SigManager& operator=(SigManager&&) = delete; concord::crypto::SignatureAlgorithm getMainKeyAlgorithm() const; - concord::crypto::ISigner* getLatestReplicaSigner() const; + std::shared_ptr getCurrentReplicaSigner() const; + std::shared_ptr getLastReplicaSigner() const; const concord::crypto::IVerifier& getVerifier(PrincipalId otherPrincipal) const; std::string getClientsPublicKeys(); - std::string getPublicKeyOfVerifier(uint32_t id) const { - if (!verifiers_.count(id)) return std::string(); - return verifiers_.at(id)->getPubKey(); - } + + // Hex format + std::pair getMyLatestPublicKey() const; + + // Used by AsyncTLSConnection to verify tls certificates which replicas sign using their main key + // Up to replicaIdentityHistoryCount keys are returned by replicas + std::array getPublicKeyOfVerifier(uint32_t id) const; // Used only by replicas std::string getSelfPrivKey() const; void setReplicaLastExecutedSeq(SeqNum seq); - SeqNum getReplicaLastExecutedSeq(); + SeqNum getReplicaLastExecutedSeq() const; - bool verifyOwnSignature(const concord::Byte* data, - size_t dataLength, - const concord::Byte* expectedSignature) const; + bool verifyOwnSignature(const concord::Byte* data, size_t dataLength, const concord::Byte* expectedSignature) const; protected: static constexpr uint16_t updateMetricsAggregatorThresh = 1000; @@ -143,6 +147,7 @@ class SigManager { bool clientTransactionSigningEnabled_ = true; const ReplicasInfo& replicasInfo_; + // The ownership model of a SigManager object depends on its use static std::shared_ptr s_sm; struct Metrics { diff --git a/bftengine/src/bftengine/SysConsts.hpp b/bftengine/src/bftengine/SysConsts.hpp index fd90063d21..90005938c3 100644 --- a/bftengine/src/bftengine/SysConsts.hpp +++ b/bftengine/src/bftengine/SysConsts.hpp @@ -130,3 +130,5 @@ constexpr uint32_t MaxSizeOfPrivateKey = 1024; // TODO(GG): should be checked constexpr uint32_t MaxSizeOfPublicKey = 1024; // TODO(GG): should be checked static const std::string secFilePrefix = "gen-sec"; + +static constexpr const size_t replicaIdentityHistoryCount = 2; \ No newline at end of file diff --git a/bftengine/src/bftengine/ValidationOnlyIdentityManager.cpp b/bftengine/src/bftengine/ValidationOnlyIdentityManager.cpp new file mode 100644 index 0000000000..b8617382d9 --- /dev/null +++ b/bftengine/src/bftengine/ValidationOnlyIdentityManager.cpp @@ -0,0 +1,52 @@ +// Concord +// +// Copyright (c) 2020-2022 VMware, Inc. All Rights Reserved. +// +// This product is licensed to you under the Apache 2.0 license (the "License"). +// You may not use this product except in compliance with the Apache 2.0 +// License. +// +// This product may include a number of subcomponents with separate copyright +// notices and license terms. Your use of these subcomponents is subject to the +// terms and conditions of the subcomponent's license, as noted in the LICENSE +// file. +#include "ValidationOnlyIdentityManager.hpp" + +namespace bftEngine::impl { + +ValidationOnlyIdentityManager::ValidationOnlyIdentityManager( + PrincipalId myId, + const std::vector>& publickeys, + const std::map& publicKeysMapping, + const ReplicasInfo& replicasInfo) + : SigManager(myId, + {"", concord::crypto::KeyFormat::HexaDecimalStrippedFormat}, + publickeys, + publicKeysMapping, + false, + replicasInfo) {} + +bool ValidationOnlyIdentityManager::verifySig( + PrincipalId pid, const concord::Byte* data, size_t dataLength, const concord::Byte* sig, uint16_t sigLength) const { + return verifyNonReplicaSig(pid, data, dataLength, sig, sigLength); +} + +// This method is assumed to be called by a single thread +std::shared_ptr ValidationOnlyIdentityManager::init( + PrincipalId myId, + const std::set>& publicKeysOfReplicas, + const ReplicasInfo& replicasInfo) { + std::vector> publicKeysWithFormat(publicKeysOfReplicas.size()); + std::map publicKeysMapping; + for (auto& [id, key] : publicKeysOfReplicas) { + publicKeysWithFormat[id] = {key, concord::crypto::KeyFormat::HexaDecimalStrippedFormat}; + publicKeysMapping.emplace(id, id); + } + + auto manager = + std::make_shared(myId, publicKeysWithFormat, publicKeysMapping, replicasInfo); + SigManager::reset(manager); + return manager; +} + +} // namespace bftEngine::impl diff --git a/bftengine/src/bftengine/ValidationOnlyIdentityManager.hpp b/bftengine/src/bftengine/ValidationOnlyIdentityManager.hpp new file mode 100644 index 0000000000..878251eefe --- /dev/null +++ b/bftengine/src/bftengine/ValidationOnlyIdentityManager.hpp @@ -0,0 +1,40 @@ +// Concord +// +// Copyright (c) 2020-2022 VMware, Inc. All Rights Reserved. +// +// This product is licensed to you under the Apache 2.0 license (the "License"). +// You may not use this product except in compliance with the Apache 2.0 +// License. +// +// This product may include a number of subcomponents with separate copyright +// notices and license terms. Your use of these subcomponents is subject to the +// terms and conditions of the subcomponent's license, as noted in the LICENSE +// file. +#include "SigManager.hpp" +#pragma once + +namespace bftEngine::impl { +/* This class is a hack to enable the validation of replica signatures + using fixed keys without initializing a CryptoManager object. + Messages such as CheckpointMsg use a global singleton to expose their validation logic, + thus an instance of this class can be used to succeed in performing the validation. +*/ +class ValidationOnlyIdentityManager : public SigManager { + public: + ValidationOnlyIdentityManager(PrincipalId myId, + const std::vector>& publickeys, + const std::map& publicKeysMapping, + const ReplicasInfo& replicasInfo); + + bool verifySig(PrincipalId pid, + const concord::Byte* data, + size_t dataLength, + const concord::Byte* sig, + uint16_t sigLength) const override; + + static std::shared_ptr init( + PrincipalId myId, + const std::set>& publicKeysOfReplicas, + const ReplicasInfo& replicasInfo); +}; +} // namespace bftEngine::impl diff --git a/bftengine/src/bftengine/ViewsManager.cpp b/bftengine/src/bftengine/ViewsManager.cpp index 621214d2ff..d7659a14ba 100644 --- a/bftengine/src/bftengine/ViewsManager.cpp +++ b/bftengine/src/bftengine/ViewsManager.cpp @@ -45,7 +45,7 @@ uint32_t ViewsManager::PrevViewInfo::maxSize() { ViewsManager::ViewsManager(const ReplicasInfo* const r) : replicasInfo(r), - N(r->numberOfReplicas()), + N(r->getNumberOfReplicas()), F(r->fVal()), C(r->cVal()), myId(r->myId()), diff --git a/bftengine/src/bftengine/messages/CheckpointMsg.cpp b/bftengine/src/bftengine/messages/CheckpointMsg.cpp index f7494987e9..691783f769 100644 --- a/bftengine/src/bftengine/messages/CheckpointMsg.cpp +++ b/bftengine/src/bftengine/messages/CheckpointMsg.cpp @@ -43,10 +43,16 @@ CheckpointMsg::CheckpointMsg(ReplicaId genReplica, void CheckpointMsg::sign() { auto sigManager = SigManager::instance(); - sigManager->sign(body(), sizeof(Header), body() + sizeof(Header) + spanContextSize()); + sigManager->sign(b()->seqNum, body(), sizeof(Header), body() + sizeof(Header) + spanContextSize()); } -void CheckpointMsg::validate(const ReplicasInfo& repInfo) const { +std::string_view CheckpointMsg::getDataBytes() const { return std::string_view{body(), sizeof(Header)}; } +std::string_view CheckpointMsg::getSignatureBytes() const { + auto sigLen = SigManager::instance()->getSigLength(idOfGeneratedReplica()); + return std::string_view{body() + sizeof(Header) + spanContextSize(), sigLen}; +} + +void CheckpointMsg::validateSize(const ReplicasInfo& repInfo) const { ConcordAssert(type() == MsgCode::Checkpoint); ConcordAssert(senderId() != repInfo.myId()); @@ -63,14 +69,21 @@ void CheckpointMsg::validate(const ReplicasInfo& repInfo) const { if (size() < sizeof(Header) + spanContextSize() + sigLen) { throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": size")); } - - if (!sigManager->verifySig(idOfGeneratedReplica(), - std::string_view{body(), sizeof(Header)}, - std::string_view{body() + sizeof(Header) + spanContextSize(), sigLen})) - throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": verifySig")); // TODO(GG): consider to protect against messages that are larger than needed (here and in other messages) } +void CheckpointMsg::validate(const ReplicasInfo& repInfo) const { + constexpr bool validateSignature = true; + validate(repInfo, validateSignature); +} + +void CheckpointMsg::validate(const ReplicasInfo& repInfo, bool validateSignature) const { + validateSize(repInfo); + if (validateSignature && !SigManager::instance()->verifySig(idOfGeneratedReplica(), getDataBytes(), getSignatureBytes())) { + throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": verifySig(replica)")); + } +} + bool CheckpointMsg::equivalent(const CheckpointMsg* a, const CheckpointMsg* b) { bool equal = (a->seqNumber() == b->seqNumber()) && (a->stateDigest() == b->stateDigest()) && (a->reservedPagesDigest() == b->reservedPagesDigest()) && (a->rvbDataDigest() == b->rvbDataDigest()) && diff --git a/bftengine/src/bftengine/messages/CheckpointMsg.hpp b/bftengine/src/bftengine/messages/CheckpointMsg.hpp index dc2689eb4b..b6eea98a44 100644 --- a/bftengine/src/bftengine/messages/CheckpointMsg.hpp +++ b/bftengine/src/bftengine/messages/CheckpointMsg.hpp @@ -55,7 +55,20 @@ class CheckpointMsg : public MessageBase { void setEpochNumber(const EpochNum& e) { b()->epochNum = e; } + // The validation of checkpoints is used both by replicas and by external tools + // such as the integrity checker. This implementation is that of the replica. void validate(const ReplicasInfo& repInfo) const override; + /** + * + * @param repInfo + * @param validateSignature - If set to true, the signature of the message is validated, + * replicas skip the signature validation when deciding whether to enter state transfer + */ + void validate(const ReplicasInfo& repInfo, bool validateSignature) const; + void validateSize(const ReplicasInfo& repInfo) const; + + std::string_view getDataBytes() const; + std::string_view getSignatureBytes() const; bool shouldValidateAsync() const override { return true; } diff --git a/bftengine/src/bftengine/messages/ClientRequestMsg.cpp b/bftengine/src/bftengine/messages/ClientRequestMsg.cpp index d5f4984b8f..4be5db2e04 100644 --- a/bftengine/src/bftengine/messages/ClientRequestMsg.cpp +++ b/bftengine/src/bftengine/messages/ClientRequestMsg.cpp @@ -142,6 +142,9 @@ void ClientRequestMsg::validateImp(const ReplicasInfo& repInfo) const { if (((header->flags & RECONFIG_FLAG) != 0 || (header->flags & INTERNAL_FLAG) != 0) && (repInfo.isIdOfReplica(clientId) || repInfo.isIdOfPeerRoReplica(clientId))) { // Allow every reconfiguration/internal message from replicas (it will be verified in the reconfiguration handler) + LOG_INFO(CNSUS, + "Reconfig/Internal replica message not validated" + << KVLOG(clientId, header->flags & RECONFIG_FLAG, header->flags & INTERNAL_FLAG)); return; } if (!repInfo.isValidPrincipalId(clientId)) { diff --git a/bftengine/src/bftengine/messages/MsgCode.hpp b/bftengine/src/bftengine/messages/MsgCode.hpp index 91edb31974..0d4d063873 100644 --- a/bftengine/src/bftengine/messages/MsgCode.hpp +++ b/bftengine/src/bftengine/messages/MsgCode.hpp @@ -24,26 +24,26 @@ class MsgCode { None = 0, PrePrepare = 100, - PreparePartial, - PrepareFull, - CommitPartial, - CommitFull, - StartSlowCommit, - PartialCommitProof, - FullCommitProof, - PartialExecProof, - FullExecProof, - SimpleAck, - ViewChange, - NewView, - Checkpoint, - AskForCheckpoint, - ReplicaStatus, - ReqMissingData, - StateTransfer, - ReplicaAsksToLeaveView, - ReplicaRestartReady, - ReplicasRestartReadyProof, + PreparePartial = 101, + PrepareFull = 102, + CommitPartial = 103, + CommitFull = 104, + StartSlowCommit = 105, + PartialCommitProof = 106, + FullCommitProof = 107, + PartialExecProof = 108, + FullExecProof = 109, + SimpleAck = 110, + ViewChange = 111, + NewView = 112, + Checkpoint = 113, + AskForCheckpoint = 114, + ReplicaStatus = 115, + ReqMissingData = 116, + StateTransfer = 117, + ReplicaAsksToLeaveView = 118, + ReplicaRestartReady = 119, + ReplicasRestartReadyProof = 120, ClientPreProcessRequest = 500, PreProcessRequest, diff --git a/bftengine/src/bftengine/messages/ReplicaAsksToLeaveViewMsg.cpp b/bftengine/src/bftengine/messages/ReplicaAsksToLeaveViewMsg.cpp index 4eeb71192f..83049c1291 100644 --- a/bftengine/src/bftengine/messages/ReplicaAsksToLeaveViewMsg.cpp +++ b/bftengine/src/bftengine/messages/ReplicaAsksToLeaveViewMsg.cpp @@ -46,7 +46,7 @@ ReplicaAsksToLeaveViewMsg* ReplicaAsksToLeaveViewMsg::create(ReplicaId senderId, std::memcpy(position, spanContext.data().data(), spanContext.data().size()); position += spanContext.data().size(); - sigManager->sign(m->body(), sizeof(Header), position); + sigManager->sign(sigManager->getReplicaLastExecutedSeq(), m->body(), sizeof(Header), position); return m; } @@ -64,7 +64,7 @@ void ReplicaAsksToLeaveViewMsg::validate(const ReplicasInfo& repInfo) const { if (!sigManager->verifySig(idOfGeneratedReplica(), std::string_view{body(), sizeof(Header)}, std::string_view{body() + totalSize, sigLen})) - throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": verifySig")); + throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": verifySigAsReplica")); } } // namespace impl diff --git a/bftengine/src/bftengine/messages/ReplicaRestartReadyMsg.cpp b/bftengine/src/bftengine/messages/ReplicaRestartReadyMsg.cpp index 86155e0de8..55a7152b3d 100644 --- a/bftengine/src/bftengine/messages/ReplicaRestartReadyMsg.cpp +++ b/bftengine/src/bftengine/messages/ReplicaRestartReadyMsg.cpp @@ -42,17 +42,20 @@ ReplicaRestartReadyMsg::ReplicaRestartReadyMsg(ReplicaId srcReplicaId, } ReplicaRestartReadyMsg* ReplicaRestartReadyMsg::create(ReplicaId senderId, - SeqNum s, + SeqNum seqNumToStopAt, Reason r, const std::string& extraData, const concordUtils::SpanContext& spanContext) { auto sigManager = SigManager::instance(); const size_t sigLen = sigManager->getMySigLength(); - ReplicaRestartReadyMsg* m = new ReplicaRestartReadyMsg(senderId, s, sigLen, r, extraData, spanContext); + ReplicaRestartReadyMsg* m = new ReplicaRestartReadyMsg(senderId, seqNumToStopAt, sigLen, r, extraData, spanContext); auto dataSize = sizeof(Header) + m->getExtraDataLength() + spanContext.data().size(); auto position = m->body() + dataSize; - sigManager->sign(reinterpret_cast(m->body()), dataSize, reinterpret_cast(position)); + sigManager->sign(seqNumToStopAt, + reinterpret_cast(m->body()), + dataSize, + reinterpret_cast(position)); //+-----------+-----------+----------+ //| Header | extraData | Signature| //+-----------+-----------+----------+ @@ -77,7 +80,7 @@ void ReplicaRestartReadyMsg::validate(const ReplicasInfo& repInfo) const { if (!sigManager->verifySig( idOfSenderReplica, std::string_view{body(), dataSize}, std::string_view{body() + dataSize, sigLen})) - throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": verifySig")); + throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": verifySig(replica)")); } } // namespace impl diff --git a/bftengine/src/bftengine/messages/ReplicaRestartReadyMsg.hpp b/bftengine/src/bftengine/messages/ReplicaRestartReadyMsg.hpp index a668d4ba8b..0bf731f47e 100644 --- a/bftengine/src/bftengine/messages/ReplicaRestartReadyMsg.hpp +++ b/bftengine/src/bftengine/messages/ReplicaRestartReadyMsg.hpp @@ -49,7 +49,7 @@ class ReplicaRestartReadyMsg : public MessageBase { char* signatureBody() const { return body() + sizeof(Header) + spanContextSize(); } static ReplicaRestartReadyMsg* create(ReplicaId senderId, - SeqNum s, + SeqNum seqNumToStopAt, Reason r, const std::string& extraData, const concordUtils::SpanContext& spanContext = {}); diff --git a/bftengine/src/bftengine/messages/ReplicasRestartReadyProofMsg.cpp b/bftengine/src/bftengine/messages/ReplicasRestartReadyProofMsg.cpp index b2455653ef..a3c81cb8c7 100644 --- a/bftengine/src/bftengine/messages/ReplicasRestartReadyProofMsg.cpp +++ b/bftengine/src/bftengine/messages/ReplicasRestartReadyProofMsg.cpp @@ -85,7 +85,7 @@ void ReplicasRestartReadyProofMsg::validate(const ReplicasInfo& repInfo) const { uint16_t sigLen = sigManager->getSigLength(idOfGeneratedReplica()); if (size() < dataLength) throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": size")); - if (elementsCount() < (repInfo.numberOfReplicas() - repInfo.fVal())) + if (elementsCount() < (repInfo.getNumberOfReplicas() - repInfo.fVal())) throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": quorum")); if (!checkElements(repInfo, sigLen)) // check elements in message throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": check elements in message")); diff --git a/bftengine/src/bftengine/messages/ViewChangeMsg.cpp b/bftengine/src/bftengine/messages/ViewChangeMsg.cpp index 704ea1ea57..8379e0c6ea 100644 --- a/bftengine/src/bftengine/messages/ViewChangeMsg.cpp +++ b/bftengine/src/bftengine/messages/ViewChangeMsg.cpp @@ -162,7 +162,9 @@ void ViewChangeMsg::finalizeMessage() { // | Message Body | // +------------------------------------------+ - sigManager->sign(body(), bodySize, body() + bodySize); + // TODO: since there are multiple message with different sequences, make sure that lastStable() + // is the correct value + sigManager->sign(lastStable(), body(), bodySize, body() + bodySize); bool b = checkElements((uint16_t)sigSize) && checkComplaints((uint16_t)sigSize); @@ -181,7 +183,7 @@ void ViewChangeMsg::validate(const ReplicasInfo& repInfo) const { if (size() < (dataLength + sigLen)) throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": size")); if (!sigManager->verifySig( idOfGeneratedReplica(), std::string_view{body(), dataLength}, std::string_view{body() + dataLength, sigLen})) - throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": verifySig")); + throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": verifySig(replica)")); if (!checkElements(sigLen)) // check elements in message throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": check elements in message")); if (!checkComplaints(sigLen)) // check list of complaints diff --git a/bftengine/src/preprocessor/RequestProcessingState.cpp b/bftengine/src/preprocessor/RequestProcessingState.cpp index b6a12ea1c9..13581a628d 100644 --- a/bftengine/src/preprocessor/RequestProcessingState.cpp +++ b/bftengine/src/preprocessor/RequestProcessingState.cpp @@ -106,7 +106,10 @@ void RequestProcessingState::handlePrimaryPreProcessed(const char *preProcessRes if (preProcessResult != OperationResult::SUCCESS) setupPreProcessResultData(preProcessResult); auto sm = SigManager::instance(); std::vector sig(sm->getMySigLength()); - sm->sign(primaryPreProcessResultHash_.data(), primaryPreProcessResultHash_.size(), sig.data()); + sm->sign(sm->getReplicaLastExecutedSeq(), + primaryPreProcessResultHash_.data(), + primaryPreProcessResultHash_.size(), + sig.data()); if (!preProcessingResultHashes_[primaryPreProcessResultHash_] .emplace(std::move(sig), myReplicaId_, preProcessResult) .second) { @@ -231,7 +234,10 @@ void RequestProcessingState::modifyPrimaryResult( primaryPreProcessResultHash_ = result.second; auto sm = SigManager::instance(); std::vector sig(sm->getMySigLength()); - sm->sign(primaryPreProcessResultHash_.data(), primaryPreProcessResultHash_.size(), sig.data()); + sm->sign(sm->getReplicaLastExecutedSeq(), + primaryPreProcessResultHash_.data(), + primaryPreProcessResultHash_.size(), + sig.data()); if (!preProcessingResultHashes_[primaryPreProcessResultHash_] .emplace(std::move(sig), myReplicaId_, primaryPreProcessResult_) .second) { diff --git a/bftengine/src/preprocessor/messages/PreProcessReplyMsg.cpp b/bftengine/src/preprocessor/messages/PreProcessReplyMsg.cpp index 2d9552a6b0..9f9013ddf1 100644 --- a/bftengine/src/preprocessor/messages/PreProcessReplyMsg.cpp +++ b/bftengine/src/preprocessor/messages/PreProcessReplyMsg.cpp @@ -87,7 +87,7 @@ void PreProcessReplyMsg::validate(const ReplicasInfo& repInfo) const { concord::crypto::SHA3_256::SIZE_IN_BYTES, reinterpret_cast(msgBody()) + headerSize, sigLen)) - throw runtime_error(__PRETTY_FUNCTION__ + string(": verifySig failed")); + throw runtime_error(__PRETTY_FUNCTION__ + string(": verifySig(replica) failed")); } } // namespace preprocessor @@ -144,6 +144,7 @@ void PreProcessReplyMsg::setupMsgBody(const char* preProcessResultBuf, const string& reqCid) { uint16_t sigSize = 0; auto sigManager = SigManager::instance(); + sigSize = sigManager->getMySigLength(); // Calculate pre-process result hash auto hash = PreProcessResultHashCreator::create(preProcessResultBuf, @@ -154,8 +155,9 @@ void PreProcessReplyMsg::setupMsgBody(const char* preProcessResultBuf, memcpy(msgBody()->resultsHash, hash.data(), concord::crypto::SHA3_256::SIZE_IN_BYTES); { concord::diagnostics::TimeRecorder scoped_timer(*preProcessorHistograms_->signPreProcessReplyHash); - sigManager->sign(hash.data(), - concord::crypto::SHA3_256::SIZE_IN_BYTES, + sigManager->sign(sigManager->getReplicaLastExecutedSeq(), + hash.data(), + concord::crypto::openssl::SHA3_256::SIZE_IN_BYTES, reinterpret_cast(body() + sizeof(Header))); } setLeftMsgParams(reqCid, sigSize); diff --git a/bftengine/src/preprocessor/messages/PreProcessResultMsg.cpp b/bftengine/src/preprocessor/messages/PreProcessResultMsg.cpp index 12948d459c..df1641db43 100644 --- a/bftengine/src/preprocessor/messages/PreProcessResultMsg.cpp +++ b/bftengine/src/preprocessor/messages/PreProcessResultMsg.cpp @@ -81,9 +81,8 @@ std::optional PreProcessResultMsg::validatePreProcessResultSignatur for (const auto& sig : sigs) { bool verificationResult = false; if (myReplicaId == sig.sender_replica) { - std::vector mySignature(sigManager_->getMySigLength(), '\0'); - sigManager_->sign(hash.data(), hash.size(), mySignature.data()); - verificationResult = mySignature == sig.signature; + // TODO: why didn't we just validate this signature normally? + verificationResult = sigManager_->verifyOwnSignature(hash.data(), hash.size(), sig.signature.data()); } else { verificationResult = sigManager_->verifySig( sig.sender_replica, hash.data(), hash.size(), sig.signature.data(), sig.signature.size()); diff --git a/bftengine/src/preprocessor/tests/CMakeLists.txt b/bftengine/src/preprocessor/tests/CMakeLists.txt index 55b3a2babd..dec466331b 100644 --- a/bftengine/src/preprocessor/tests/CMakeLists.txt +++ b/bftengine/src/preprocessor/tests/CMakeLists.txt @@ -1,9 +1,9 @@ find_package(GTest REQUIRED) -add_executable(preprocessor_test preprocessor_test.cpp ) +add_executable(preprocessor_test preprocessor_test.cpp ../../../tests/messages/helper.cpp) add_test(preprocessor_test preprocessor_test) -target_include_directories(preprocessor_test PUBLIC .. ../../../..) +target_include_directories(preprocessor_test PUBLIC .. ../../.. ../../../..) target_link_libraries(preprocessor_test PUBLIC GTest::Main diff --git a/bftengine/src/preprocessor/tests/messages/ClientPreProcessRequestMsg_test.cpp b/bftengine/src/preprocessor/tests/messages/ClientPreProcessRequestMsg_test.cpp index d09b8c9b3f..8c6a4f4ea8 100644 --- a/bftengine/src/preprocessor/tests/messages/ClientPreProcessRequestMsg_test.cpp +++ b/bftengine/src/preprocessor/tests/messages/ClientPreProcessRequestMsg_test.cpp @@ -23,7 +23,7 @@ class ClientPreprocessRequestMsgTestFixture : public ::testing::Test { ReplicaConfig& config; ReplicasInfo replicaInfo; - std::unique_ptr sigManager; + std::shared_ptr sigManager; }; TEST_F(ClientPreprocessRequestMsgTestFixture, create_and_compare) { diff --git a/bftengine/src/preprocessor/tests/messages/PreProcessReplyMsg_test.cpp b/bftengine/src/preprocessor/tests/messages/PreProcessReplyMsg_test.cpp index e8ae00d7d4..d34cc31966 100644 --- a/bftengine/src/preprocessor/tests/messages/PreProcessReplyMsg_test.cpp +++ b/bftengine/src/preprocessor/tests/messages/PreProcessReplyMsg_test.cpp @@ -38,7 +38,7 @@ class PreProcessReplyMsgTestFixture : public testing::Test { ReplicaConfig& config; ReplicasInfo replicaInfo; - std::unique_ptr sigManager; + std::shared_ptr sigManager; PreProcessorRecorder preProcessorRecorder; }; @@ -110,7 +110,7 @@ TEST_F(PreProcessReplyMsgTestFixture, getResultHashSignature) { const auto hash = PreProcessResultHashCreator::create(preProcessResultBuf, preProcessResultBufLen, opResult, clientId, reqSeqNum); auto expected_signature = std::vector(sigManager->getMySigLength()); - sigManager->sign(hash.data(), sizeof(hash), expected_signature.data()); + sigManager->sign(0, hash.data(), sizeof(hash), expected_signature.data()); EXPECT_THAT(expected_signature, testing::ContainerEq(preProcessReplyMsg.getResultHashSignature())); clearDiagnosticsHandlers(); } diff --git a/bftengine/src/preprocessor/tests/messages/PreProcessResultMsg_test.cpp b/bftengine/src/preprocessor/tests/messages/PreProcessResultMsg_test.cpp index e9e9866b73..cd8fef6717 100644 --- a/bftengine/src/preprocessor/tests/messages/PreProcessResultMsg_test.cpp +++ b/bftengine/src/preprocessor/tests/messages/PreProcessResultMsg_test.cpp @@ -36,13 +36,22 @@ bftEngine::ReplicaConfig& createReplicaConfigWithExtClient(uint16_t fVal, return config; } -bftEngine::impl::SigManager* createSigManagerWithSigning( +std::shared_ptr createSigManagerWithTransactionSigning( size_t myId, std::string& myPrivateKey, concord::crypto::KeyFormat replicasKeysFormat, std::set>& publicKeysOfReplicas, const std::set>>* publicKeysOfClients, ReplicasInfo& replicasInfo) { + std::vector replicaPublicKeys(replicasInfo.getNumberOfReplicas()); + + for (auto& [id, key] : publicKeysOfReplicas) { + ConcordAssert(replicasInfo.isIdOfReplica(id)); + replicaPublicKeys[id] = key; + } + + bftEngine::CryptoManager::init(std::make_unique(myId, replicaPublicKeys, myPrivateKey)); + return SigManager::init(myId, myPrivateKey, publicKeysOfReplicas, @@ -68,14 +77,14 @@ std::vector copyAsBytes(const std::vector& vec) { return reinterpret_cast&>(vec); } -std::pair> getProcessResultSigBuff(const std::unique_ptr& sigManager, +std::pair> getProcessResultSigBuff(const std::shared_ptr& sigManager, const MsgParams& p, const int sigCount) { auto msgSigSize = sigManager->getMySigLength(); std::vector msgSig(msgSigSize, 0); auto hash = PreProcessResultHashCreator::create( p.result, sizeof(p.result), OperationResult::SUCCESS, p.senderId, p.reqSeqNum); - sigManager->sign(reinterpret_cast(hash.data()), hash.size(), msgSig.data()); + sigManager->sign(0, reinterpret_cast(hash.data()), hash.size(), msgSig.data()); // for simplicity, copy the same signatures std::set resultSigs; @@ -113,15 +122,15 @@ class PreProcessResultMsgTestFixture : public testing::Test { PreProcessResultMsgTestFixture() : config{createReplicaConfigWithExtClient(1, 0, true)}, replicaInfo{config, false, false}, - sigManager(createSigManagerWithSigning(config.replicaId, - config.replicaPrivateKey, - concord::crypto::KeyFormat::HexaDecimalStrippedFormat, - config.publicKeysOfReplicas, - &config.publicKeysOfClients, - replicaInfo)) {} + sigManager(createSigManagerWithTransactionSigning(config.replicaId, + config.replicaPrivateKey, + concord::crypto::KeyFormat::HexaDecimalStrippedFormat, + config.publicKeysOfReplicas, + &config.publicKeysOfClients, + replicaInfo)) {} ReplicaConfig& config; ReplicasInfo replicaInfo; - std::unique_ptr sigManager; + std::shared_ptr sigManager; std::unique_ptr createMessage(const MsgParams& p, const int sigCount) { auto resultSigsBuf = getProcessResultSigBuff(sigManager, p, sigCount); @@ -131,7 +140,7 @@ class PreProcessResultMsgTestFixture : public testing::Test { auto hash = PreProcessResultHashCreator::create( p.result, sizeof(p.result), OperationResult::SUCCESS, p.senderId, p.reqSeqNum); - sigManager->sign(reinterpret_cast(hash.data()), hash.size(), msgSig.data()); + sigManager->sign(0, reinterpret_cast(hash.data()), hash.size(), msgSig.data()); // For simplicity, copy the same signatures std::list resultSigs; @@ -162,16 +171,16 @@ class PreProcessResultMsgTxSigningOffTestFixture : public testing::Test { PreProcessResultMsgTxSigningOffTestFixture() : config{createReplicaConfigWithExtClient(1, 0, false)}, replicaInfo{config, false, false}, - sigManager(createSigManagerWithSigning(config.replicaId, - config.replicaPrivateKey, - concord::crypto::KeyFormat::HexaDecimalStrippedFormat, - config.publicKeysOfReplicas, - &config.publicKeysOfClients, - replicaInfo)) {} + sigManager(createSigManagerWithTransactionSigning(config.replicaId, + config.replicaPrivateKey, + concord::crypto::KeyFormat::HexaDecimalStrippedFormat, + config.publicKeysOfReplicas, + &config.publicKeysOfClients, + replicaInfo)) {} ReplicaConfig& config; ReplicasInfo replicaInfo; - std::unique_ptr sigManager; + std::shared_ptr sigManager; std::unique_ptr createMessage(const MsgParams& p, const int sigCount) { auto resultSigsBuf = getProcessResultSigBuff(sigManager, p, sigCount); @@ -260,7 +269,7 @@ TEST_F(PreProcessResultMsgTestFixture, MsgDeserialisation) { std::vector msgSig(msgSigSize, 0); auto hash = PreProcessResultHashCreator::create( msg->requestBuf(), msg->requestLength(), OperationResult::SUCCESS, params.senderId, params.reqSeqNum); - sigManager->sign(reinterpret_cast(hash.data()), hash.size(), msgSig.data()); + sigManager->sign(0, reinterpret_cast(hash.data()), hash.size(), msgSig.data()); auto i = 0; for (const auto& s : sigs) { ASSERT_EQ(s.sender_replica, i++); @@ -294,7 +303,7 @@ TEST_F(PreProcessResultMsgTestFixture, MsgDeserialisationFromBase) { std::vector msgSig(msgSigSize, 0); auto hash = PreProcessResultHashCreator::create( msg->requestBuf(), msg->requestLength(), OperationResult::SUCCESS, params.senderId, params.reqSeqNum); - sigManager->sign(reinterpret_cast(hash.data()), hash.size(), msgSig.data()); + sigManager->sign(0, reinterpret_cast(hash.data()), hash.size(), msgSig.data()); auto i = 0; for (const auto& s : sigs) { ASSERT_EQ(s.sender_replica, i++); diff --git a/bftengine/src/preprocessor/tests/preprocessor_test.cpp b/bftengine/src/preprocessor/tests/preprocessor_test.cpp index ea4e65f951..99194f80c3 100644 --- a/bftengine/src/preprocessor/tests/preprocessor_test.cpp +++ b/bftengine/src/preprocessor/tests/preprocessor_test.cpp @@ -25,6 +25,7 @@ #include "gtest/gtest.h" #include "threshsign/eddsa/EdDSAMultisigFactory.h" #include "CryptoManager.hpp" +#include "tests/messages/helper.hpp" #include "tests/config/test_comm_config.hpp" #include "bftengine/MsgsCommunicator.hpp" @@ -166,7 +167,7 @@ class DummyPreProcessor : public PreProcessor { }; // clang-format off -unordered_map replicaPrivKeys = { +const unordered_map replicaPrivKeys = { {replica_0, "61498efe1764b89357a02e2887d224154006ceacf26269f8695a4af561453eef"}, {replica_1, "247a74ab3620ec6b9f5feab9ee1f86521da3fa2804ad45bb5bf2c5b21ef105bc"}, {replica_2, "fb539bc3d66deda55524d903da26dbec1f4b6abf41ec5db521e617c64eb2c341"}, @@ -174,7 +175,7 @@ unordered_map replicaPrivKeys = { {replica_4, "f2f3d43da68329bfe31419636072e27cfd1a8fff259be4bfada667080eb00556"} }; -unordered_map replicaPubKeys = { +const unordered_map replicaPubKeys = { {replica_0, "386f4fb049a5d8bb0706d3793096c8f91842ce380dfc342a2001d50dfbc901f4"}, {replica_1, "3f9e7dbde90477c24c1bacf14e073a356c1eca482d352d9cc0b16560a4e7e469"}, {replica_2, "2311c6013ff657844669d8b803b2e1ed33fe06eed445f966a800a8fbb8d790e8"}, @@ -183,25 +184,6 @@ unordered_map replicaPubKeys = { }; // clang-format on - -class PPCryptoSystem : public Cryptosystem { - public: - PPCryptoSystem(NodeIdType id) : id_{id} {} - IThresholdVerifier *createThresholdVerifier(uint16_t threshold = 0) override { - vector publicKeys; - for (int i = 0; i < static_cast(replicaPubKeys.size()); i++) { - publicKeys.push_back(replicaPubKeys.at(i)); - } - return factory_.newVerifier(threshold, publicKeys.size(), "", publicKeys); - } - IThresholdSigner *createThresholdSigner() override { return factory_.newSigner(id_ + 1, replicaPrivKeys.at(id_).c_str()); } - - private: - NodeIdType id_; - EdDSAMultisigFactory factory_; -}; - - void switchReplicaContext(NodeIdType id) { LOG_INFO(GL, "Switching replica context to replica " << id); SigManager::reset(sigManager[id]); @@ -219,22 +201,25 @@ void setUpConfiguration_4() { replicaConfig.numOfExternalClients = 15; replicaConfig.clientBatchingEnabled = true; - for (NodeIdType i = replica_0; i <= replica_3; ++i) { - replicaConfig.publicKeysOfReplicas.insert(pair(i, replicaPubKeys[i])); + std::vector publicKeysVector; + for (NodeIdType i = replica_0; i < replica_4; ++i) { + replicaConfig.publicKeysOfReplicas.emplace(i, replicaPubKeys.at(i)); + publicKeysVector.push_back(replicaPubKeys.at(i)); } - replicaConfig.replicaPrivateKey = replicaPrivKeys[replica_0]; + replicaConfig.replicaPrivateKey = replicaPrivKeys.at(replica_0); for (auto i = 0; i < replicaConfig.numReplicas; i++) { replicaConfig.replicaId = i; replicasInfo[i] = std::make_unique(replicaConfig, true, true); sigManager[i] = SigManager::init(i, - replicaPrivKeys[i], + replicaPrivKeys.at(i), replicaConfig.publicKeysOfReplicas, concord::crypto::KeyFormat::HexaDecimalStrippedFormat, nullptr, concord::crypto::KeyFormat::HexaDecimalStrippedFormat, *replicasInfo[i].get()); - cryptoManager[i] = CryptoManager::init(std::make_unique(i)); + cryptoManager[i] = + CryptoManager::init(std::make_unique(i, publicKeysVector, replicaPrivKeys.at(i))); } replicaConfig.replicaId = replica_0; } @@ -244,8 +229,8 @@ void setUpConfiguration_7() { replicaConfig.numReplicas = numOfReplicas_7; replicaConfig.fVal = fVal_7; - replicaConfig.publicKeysOfReplicas.insert(pair(replica_4, replicaPubKeys[replica_4])); - replicaConfig.replicaPrivateKey = replicaPrivKeys[replica_4]; + replicaConfig.publicKeysOfReplicas.insert(pair(replica_4, replicaPubKeys.at(replica_4))); + replicaConfig.replicaPrivateKey = replicaPrivKeys.at(replica_4); } void setUpCommunication() { @@ -290,10 +275,7 @@ void clearDiagnosticsHandlers() { class PreprocessingStateTest : public testing::Test { public: - void SetUp() override { - switchReplicaContext(replica_0); - } - + void SetUp() override { switchReplicaContext(replica_0); } }; TEST_F(PreprocessingStateTest, notEnoughRepliesReceived) { @@ -1025,7 +1007,7 @@ TEST_F(PreprocessingStateTest, rejectMsgWithInvalidView) { replica.setPrimary(true); switchReplicaContext(repInfo.myId()); - ConcordAssert(preProcessor.validateBatchReplyMsgCorrectness(batchReplyValidView)); + ConcordAssert(preProcessor.validateBatchReplyMsgCorrectness(batchReplyValidView)); ConcordAssert(!preProcessor.validateBatchReplyMsgCorrectness(batchReplyInvalidView)); replica.stop(); } diff --git a/bftengine/tests/SigManager/SigManager_test.cpp b/bftengine/tests/SigManager/SigManager_test.cpp index a3a27ac626..e3f4b2f3b3 100644 --- a/bftengine/tests/SigManager/SigManager_test.cpp +++ b/bftengine/tests/SigManager/SigManager_test.cpp @@ -20,12 +20,6 @@ using namespace std; using concord::crypto::KeyFormat; - -constexpr char KEYS_BASE_PARENT_PATH[] = "/tmp/"; -constexpr char KEYS_BASE_PATH[] = "/tmp/transaction_signing_keys"; -constexpr char PRIV_KEY_NAME[] = "privkey.pem"; -constexpr char PUB_KEY_NAME[] = "pubkey.pem"; -constexpr char KEYS_GEN_SCRIPT_PATH[] = "../../../../scripts/linux/create_concord_clients_transaction_signing_keys.sh"; constexpr size_t RANDOM_DATA_SIZE = 1000U; std::default_random_engine generator; @@ -34,34 +28,16 @@ using concord::crypto::ISigner; using concord::crypto::IVerifier; using concord::crypto::Factory; using bftEngine::ReplicaConfig; +using bftEngine::CryptoManager; using concord::crypto::SignatureAlgorithm; using concord::crypto::generateEdDSAKeyPair; -void generateKeyPairs(size_t count) { - ostringstream cmd; - - ASSERT_EQ(0, system(cmd.str().c_str())); - cmd << "rm -rf " << KEYS_BASE_PATH; - ASSERT_EQ(0, system(cmd.str().c_str())); - - cmd.str(""); - cmd.clear(); - - std::string algo; - if (ReplicaConfig::instance().replicaMsgSigningAlgo == SignatureAlgorithm::EdDSA) { - algo = "eddsa"; +std::vector> generateKeyPairs(size_t count) { + std::vector> result; + for (size_t i = 0; i < count; i++) { + result.push_back(generateEdDSAKeyPair(KeyFormat::HexaDecimalStrippedFormat)); } - cmd << KEYS_GEN_SCRIPT_PATH << " -n " << count << " -r " << PRIV_KEY_NAME << " -u " << PUB_KEY_NAME << " -o " - << KEYS_BASE_PARENT_PATH << " -a " << algo; - ASSERT_EQ(0, system(cmd.str().c_str())); -} - -void readFile(string_view path, string& keyOut) { - stringstream stream; - ifstream file(path.data()); - ASSERT_TRUE(file.good()); - stream << file.rdbuf(); - keyOut = stream.str(); + return result; } void generateRandomData(char* data, size_t len) { @@ -73,7 +49,7 @@ void generateRandomData(char* data, size_t len) { void corrupt(concord::Byte* data, size_t len) { for (size_t i{0}; i < len; ++i) { - ++data[i]; + data[i] = ~data[i]; } } @@ -113,22 +89,16 @@ TEST(SignerAndVerifierTest, LoadSignVerifyFromHexKeyPair) { } TEST(SignerAndVerifierTest, LoadSignVerifyFromPemfiles) { - string publicKeyFullPath({string(KEYS_BASE_PATH) + string("/1/") + PUB_KEY_NAME}); - string privateKeyFullPath({string(KEYS_BASE_PATH) + string("/1/") + PRIV_KEY_NAME}); - - string privKey, pubkey; char data[RANDOM_DATA_SIZE]{0}; - - generateKeyPairs(1); + auto keys = generateKeyPairs(1); + auto& [privKey, pubkey] = keys[0]; generateRandomData(data, RANDOM_DATA_SIZE); - readFile(privateKeyFullPath, privKey); - readFile(publicKeyFullPath, pubkey); - const auto signer_ = - Factory::getSigner(privKey, ReplicaConfig::instance().replicaMsgSigningAlgo, KeyFormat::PemFormat); - const auto verifier_ = - Factory::getVerifier(pubkey, ReplicaConfig::instance().replicaMsgSigningAlgo, KeyFormat::PemFormat); + const auto signer_ = Factory::getSigner( + privKey, ReplicaConfig::instance().replicaMsgSigningAlgo, KeyFormat::HexaDecimalStrippedFormat); + const auto verifier_ = Factory::getVerifier( + pubkey, ReplicaConfig::instance().replicaMsgSigningAlgo, KeyFormat::HexaDecimalStrippedFormat); // sign with replica signer. size_t expectedSignerSigLen = signer_->signatureLength(); @@ -153,126 +123,78 @@ TEST(SignerAndVerifierTest, LoadSignVerifyFromPemfiles) { ASSERT_FALSE(verifier_->verify(str_data, sig)); } -TEST(SigManagerTest, ReplicasOnlyCheckVerify) { - constexpr size_t numReplicas{4}; - constexpr PrincipalId myId{0}; - string myPrivKey; - unique_ptr signers[numReplicas]; - set> publicKeysOfReplicas; +class SigManagerTest : public ::testing::Test { + public: + SigManagerTest() : config{createReplicaConfig()}, replicaInfo{config, false, false} {} - generateKeyPairs(numReplicas); + void SetUp() override { + hexKeyPairs = generateKeyPairs(config.numReplicas); + std::set> publicKeysOfReplicas; + std::vector hexReplicaPublicKeys; - // Load signers to simulate other replicas - for (size_t i{1}; i <= numReplicas; ++i) { - string privKey, pubKey; - string privateKeyFullPath({string(KEYS_BASE_PATH) + string("/") + to_string(i) + string("/") + PRIV_KEY_NAME}); - readFile(privateKeyFullPath, privKey); - PrincipalId pid = i - 1; // folders are 1-indexed - - if (pid == myId) { - myPrivKey = privKey; - continue; + // generateKeyPairs(config.numReplicas + config.numRoReplicas + config.) + for (size_t i = 0; i < config.numReplicas; ++i) { + publicKeysOfReplicas.emplace(i, concord::crypto::EdDSAHexToPem(hexKeyPairs[i]).second); + hexReplicaPublicKeys.push_back(hexKeyPairs[i].second); } - signers[pid] = Factory::getSigner(privKey, ReplicaConfig::instance().replicaMsgSigningAlgo, KeyFormat::PemFormat); - string pubKeyFullPath({string(KEYS_BASE_PATH) + string("/") + to_string(i) + string("/") + PUB_KEY_NAME}); - readFile(pubKeyFullPath, pubKey); - publicKeysOfReplicas.insert(make_pair(pid, pubKey)); - } - ReplicasInfo replicaInfo(createReplicaConfig(), false, false); - unique_ptr sigManager(SigManager::init( - myId, myPrivKey, publicKeysOfReplicas, KeyFormat::PemFormat, nullptr, KeyFormat::PemFormat, replicaInfo)); + sigManager = SigManager::init(config.replicaId, + concord::crypto::EdDSAHexToPem(hexKeyPairs[config.replicaId]).first, + publicKeysOfReplicas, + KeyFormat::PemFormat, + nullptr, + KeyFormat::PemFormat, + replicaInfo); - for (size_t i{0}; i < numReplicas; ++i) { - const auto& signer = signers[i]; - - char data[RANDOM_DATA_SIZE]{0}; - size_t lenRetData; - size_t expectedSignerSigLen; + bftEngine::CryptoManager::init(std::make_unique( + config.replicaId, hexReplicaPublicKeys, hexKeyPairs[config.replicaId].first)); + } - if (i == myId) continue; + protected: + std::vector> hexKeyPairs; + std::shared_ptr sigManager; + const ReplicaConfig& config; + ReplicasInfo replicaInfo; +}; - // sign with replica signer (other replicas, mock) - expectedSignerSigLen = signer->signatureLength(); - std::vector sig(expectedSignerSigLen); - generateRandomData(data, RANDOM_DATA_SIZE); - std::string str_data(data, RANDOM_DATA_SIZE); - lenRetData = signer->sign(str_data, sig.data()); - ASSERT_EQ(lenRetData, expectedSignerSigLen); - - // Validate with SigManager (my replica) - ASSERT_EQ(sig.size(), sigManager->getSigLength(i)); - ASSERT_TRUE(sigManager->verifySig(i, std::string_view{data, RANDOM_DATA_SIZE}, sig)); +TEST_F(SigManagerTest, TestMySigLength) { + ASSERT_GT(sigManager->getMySigLength(), 0); +} - // change data randomally, expect failure - char data1[RANDOM_DATA_SIZE]; - std::copy(std::begin(data), std::end(data), std::begin(data1)); - corrupt(data1 + 10, 1); - ASSERT_FALSE(sigManager->verifySig(i, std::string_view{data1, RANDOM_DATA_SIZE}, sig)); +TEST_F(SigManagerTest, ReplicasOnlyCheckVerify) { + const size_t numReplicas = config.numReplicas; + std::vector> signers(numReplicas); - // change signature randomally, expect failure - corrupt(sig.data(), 1); - ASSERT_FALSE(sigManager->verifySig(i, std::string_view{data, RANDOM_DATA_SIZE}, sig)); + // Load signers to simulate other replicas + for (size_t i = 0; i < numReplicas; ++i) { + signers[i] = Factory::getSigner( + hexKeyPairs[i].first, ReplicaConfig::instance().replicaMsgSigningAlgo, KeyFormat::HexaDecimalStrippedFormat); } -} - -TEST(SigManagerTest, ReplicasOnlyCheckSign) { - constexpr size_t numReplicas{4}; - constexpr PrincipalId myId{0}; - string myPrivKey, privKey, pubKey, sig; - unique_ptr verifier; - set> publicKeysOfReplicas; - char data[RANDOM_DATA_SIZE]{0}; - size_t expectedSignerSigLen; - generateKeyPairs(numReplicas); + std::vector data(RANDOM_DATA_SIZE); + generateRandomData(data.data(), RANDOM_DATA_SIZE); - // Load my private key - string privateKeyFullPath({string(KEYS_BASE_PATH) + string("/") + to_string(1) + string("/") + PRIV_KEY_NAME}); - readFile(privateKeyFullPath, myPrivKey); + for (size_t i = 0; i < numReplicas; ++i) { + const auto& signer = signers[i]; - // Load single other replica's verifier (mock) - string pubKeyFullPath({string(KEYS_BASE_PATH) + string("/") + to_string(1) + string("/") + PUB_KEY_NAME}); - readFile(pubKeyFullPath, pubKey); + // sign with replica signer (other replicas, mock) + std::vector sig(signer->signatureLength()); + ASSERT_EQ(signer->sign(data, sig.data()), sig.size()); - verifier = Factory::getVerifier(pubKey, ReplicaConfig::instance().replicaMsgSigningAlgo, KeyFormat::PemFormat); + // Validate with SigManager (my replica) + ASSERT_EQ(sig.size(), sigManager->getSigLength(i)); + ASSERT_TRUE(sigManager->verifySig(i, data, sig)); - // load public key of other replicas, must be done for SigManager ctor - for (size_t i{2}; i <= numReplicas; ++i) { - pubKeyFullPath = string(KEYS_BASE_PATH) + string("/") + to_string(i) + string("/") + PUB_KEY_NAME; - readFile(pubKeyFullPath, pubKey); - publicKeysOfReplicas.insert(make_pair(i - 1, pubKey)); + size_t offset = i % data.size(); + corrupt(data.data() + offset, 1); + ASSERT_FALSE(sigManager->verifySig(i, data, sig)); } - - ReplicasInfo replicaInfo(createReplicaConfig(), false, false); - unique_ptr sigManager(SigManager::init( - myId, myPrivKey, publicKeysOfReplicas, KeyFormat::PemFormat, nullptr, KeyFormat::PemFormat, replicaInfo)); - // sign with SigManager - expectedSignerSigLen = sigManager->getSigLength(myId); - generateRandomData(data, RANDOM_DATA_SIZE); - sig.resize(expectedSignerSigLen); - sigManager->sign(data, RANDOM_DATA_SIZE, sig.data()); - - // Validate with replica verifier (mock) - std::string str_data(data, RANDOM_DATA_SIZE); - ASSERT_TRUE(verifier->verify(str_data, sig)); - - // change data randomally, expect failure - char data1[RANDOM_DATA_SIZE]; - std::copy(std::begin(data), std::end(data), std::begin(data1)); - corrupt(data1 + 10, 1); - std::string str_data1(data1, RANDOM_DATA_SIZE); - ASSERT_FALSE(verifier->verify(str_data1, sig)); - - // change signature randomally, expect failure - corrupt(sig.data(), 1); - ASSERT_FALSE(verifier->verify(str_data1, sig)); } // Check 1 more replica + 1200 clients on 6 additional participants // where each participant hols a client pool of 200 clients -TEST(SigManagerTest, ReplicasAndClientsCheckVerify) { +TEST(SigManagerTestWithClients, ReplicasAndClientsCheckVerify) { constexpr size_t numReplicas{7}; constexpr size_t numRoReplicas{2}; constexpr size_t numOfClientProxies{36}; // (numRoReplicas+numRoReplicas) * 4 @@ -280,54 +202,40 @@ TEST(SigManagerTest, ReplicasAndClientsCheckVerify) { constexpr size_t numBftClientsInParticipantNodes{200}; constexpr size_t totalNumberofExternalBftClients{1200}; // numOfExternaClients * numBftClientsInExternalClient constexpr PrincipalId myId{0}; - string myPrivKey; - size_t i, signerIndex{0}; + size_t signerIndex{0}; unique_ptr signers[numReplicas + numParticipantNodes]; // only external clients and consensus replicas sign set> publicKeysOfReplicas; set>> publicKeysOfClients; unordered_map principalIdToSignerIndex; - generateKeyPairs(numReplicas + numParticipantNodes); + auto keyPairs = generateKeyPairs(numReplicas + numParticipantNodes); // Load replica signers to simulate other replicas PrincipalId currPrincipalId{0}; - for (i = 1; i <= numReplicas; ++i, ++currPrincipalId) { - string privKey, pubKey; - string privateKeyFullPath({string(KEYS_BASE_PATH) + string("/") + to_string(i) + string("/") + PRIV_KEY_NAME}); - readFile(privateKeyFullPath, privKey); + for (currPrincipalId = 0; currPrincipalId < numReplicas; ++currPrincipalId) { + signers[signerIndex] = Factory::getSigner(keyPairs[currPrincipalId].first, + ReplicaConfig::instance().replicaMsgSigningAlgo, + KeyFormat::HexaDecimalStrippedFormat); - if (currPrincipalId == myId) { - myPrivKey = privKey; - continue; - } - signers[signerIndex] = - Factory::getSigner(privKey, ReplicaConfig::instance().replicaMsgSigningAlgo, KeyFormat::PemFormat); - - string pubKeyFullPath({string(KEYS_BASE_PATH) + string("/") + to_string(i) + string("/") + PUB_KEY_NAME}); - readFile(pubKeyFullPath, pubKey); - publicKeysOfReplicas.insert(make_pair(currPrincipalId, pubKey)); - principalIdToSignerIndex.insert(make_pair(currPrincipalId, signerIndex)); + publicKeysOfReplicas.insert(make_pair(currPrincipalId, keyPairs[currPrincipalId].second)); + principalIdToSignerIndex.emplace(currPrincipalId, signerIndex); ++signerIndex; } // Load another group of replica signers to simulate other clients currPrincipalId = numReplicas + numRoReplicas + numOfClientProxies; - for (; i <= numReplicas + numParticipantNodes; ++i) { - string privKey, pubKey; - string privateKeyFullPath({string(KEYS_BASE_PATH) + string("/") + to_string(i) + string("/") + PRIV_KEY_NAME}); - readFile(privateKeyFullPath, privKey); - signers[signerIndex] = - Factory::getSigner(privKey, ReplicaConfig::instance().replicaMsgSigningAlgo, KeyFormat::PemFormat); - string pubKeyFullPath({string(KEYS_BASE_PATH) + string("/") + to_string(i) + string("/") + PUB_KEY_NAME}); + for (size_t i = numReplicas; i < numReplicas + numParticipantNodes; ++i) { + signers[signerIndex] = Factory::getSigner(keyPairs[signerIndex].first, + ReplicaConfig::instance().replicaMsgSigningAlgo, + KeyFormat::HexaDecimalStrippedFormat); set principalIds; for (size_t j{0}; j < numBftClientsInParticipantNodes; ++j) { principalIds.insert(principalIds.end(), currPrincipalId); principalIdToSignerIndex.insert(make_pair(currPrincipalId, signerIndex)); ++currPrincipalId; } - readFile(pubKeyFullPath, pubKey); - publicKeysOfClients.insert(make_pair(std::move(pubKey), std::move(principalIds))); + publicKeysOfClients.insert(make_pair(keyPairs[signerIndex].second, std::move(principalIds))); ++signerIndex; } @@ -338,13 +246,19 @@ TEST(SigManagerTest, ReplicasAndClientsCheckVerify) { config.numOfExternalClients = totalNumberofExternalBftClients; ReplicasInfo replicaInfo(config, false, false); - unique_ptr sigManager(SigManager::init(myId, - myPrivKey, + shared_ptr sigManager(SigManager::init(myId, + keyPairs[config.replicaId].first, publicKeysOfReplicas, - KeyFormat::PemFormat, + KeyFormat::HexaDecimalStrippedFormat, &publicKeysOfClients, - KeyFormat::PemFormat, + KeyFormat::HexaDecimalStrippedFormat, replicaInfo)); + std::vector replicaHexPublicKeys(config.numReplicas); + for (int j = 0; j < config.numReplicas; j++) { + replicaHexPublicKeys[j] = keyPairs[j].second; + } + CryptoManager::init(std::make_unique( + config.replicaId, replicaHexPublicKeys, keyPairs[config.replicaId].second)); // principalIdToSignerIndex carries all principal ids for replica, read only replicas and bft-clients. // There are some principal Ids in the range [minKey(principalIdToSignerIndex), maxKey(principalIdToSignerIndex)] @@ -352,34 +266,35 @@ TEST(SigManagerTest, ReplicasAndClientsCheckVerify) { // In the next loop we will sign 30K times. Every iteration a random principal ID between minKey and maxKey is // generated. If the ID is valid, we locate the right signer using principalIdToSignerIndex. If not - sign with // another signer and expect a failure. - PrincipalId minPidInclusive = 1, maxPidInclusive = currPrincipalId - 1; + PrincipalId minPidInclusive = 0; + PrincipalId maxPidInclusive = currPrincipalId; std::uniform_int_distribution distribution(minPidInclusive, maxPidInclusive); - for (size_t i{0}; i < 3E4; ++i) { - char data[RANDOM_DATA_SIZE]{0}; - size_t lenRetData, expectedVerifierSigLen, expectedSignerSigLen, signerIndex; - bool expectFailure = false; - - PrincipalId signerPrincipalId = static_cast(distribution(generator)); - auto iter = principalIdToSignerIndex.find(signerPrincipalId); - if (iter != principalIdToSignerIndex.end()) { - signerIndex = iter->second; - } else { - signerIndex = 1; // sign with signer index 1, so we can check the target SigManager - expectFailure = true; + std::vector data(RANDOM_DATA_SIZE); + std::vector sig(1024u); + + for (size_t principalId = 0; principalId < maxPidInclusive; ++principalId) { + size_t lenRetData, signerOffset; + + auto iter = principalIdToSignerIndex.find(principalId); + if (iter == principalIdToSignerIndex.end()) { + ASSERT_FALSE(sigManager->hasVerifier(principalId)); + ASSERT_EQ(sigManager->getSigLength(principalId), 0); + continue; } + signerOffset = iter->second; // sign - expectedSignerSigLen = signers[signerIndex]->signatureLength(); - std::vector sig(expectedSignerSigLen); - generateRandomData(data, RANDOM_DATA_SIZE); - std::string str_data(data, RANDOM_DATA_SIZE); - lenRetData = signers[signerIndex]->sign(str_data, sig.data()); + LOG_DEBUG(GL, KVLOG(principalId, signerOffset, config.replicaId)); + auto expectedSignerSigLen = signers[signerOffset]->signatureLength(); + ASSERT_GE(sig.size(), expectedSignerSigLen); + generateRandomData(data.data(), RANDOM_DATA_SIZE); + lenRetData = signers[signerOffset]->sign(data, sig.data()); ASSERT_EQ(lenRetData, expectedSignerSigLen); // Validate with SigManager (my replica) - expectedVerifierSigLen = sigManager->getSigLength(signerPrincipalId); - ASSERT_TRUE((expectFailure && expectedVerifierSigLen == 0) || (!expectFailure && expectedVerifierSigLen > 0)); - bool signatureValid = sigManager->verifySig(signerPrincipalId, std::string_view{data, RANDOM_DATA_SIZE}, sig); - ASSERT_TRUE((expectFailure && !signatureValid) || (!expectFailure && signatureValid)); + ASSERT_TRUE(sigManager->getSigLength(principalId) > 0); + bool signatureValid = sigManager->verifySig( + principalId, data, string_view{reinterpret_cast(sig.data()), sigManager->getSigLength(principalId)}); + ASSERT_TRUE(signatureValid); } } diff --git a/bftengine/tests/clientsManager/ClientsManager_test.cpp b/bftengine/tests/clientsManager/ClientsManager_test.cpp index 68250177b4..a016ae3ffd 100644 --- a/bftengine/tests/clientsManager/ClientsManager_test.cpp +++ b/bftengine/tests/clientsManager/ClientsManager_test.cpp @@ -133,7 +133,7 @@ shared_ptr mock_secrets_manager_impl_for_key_exchange_ma unique_ptr mock_client_public_key_store_for_key_exchange_manager( new MockClientPublicKeyStore()); unique_ptr timers_for_key_exchange_manager(new Timers()); -unique_ptr sig_manager_for_key_exchange_manager; +std::shared_ptr sig_manager_for_key_exchange_manager; // Helper functions for managing and checking global objects and/or singletons the ClientsManager uses. static void resetMockReservedPages() { @@ -146,13 +146,13 @@ static void resetSigManager() { if (ReplicaConfig::instance().replicaMsgSigningAlgo == SignatureAlgorithm::EdDSA) { kReplicaPrivateKeyForTesting = generateEdDSAKeyPair().first; } - sig_manager_for_key_exchange_manager.reset(SigManager::init(kReplicaIdForTesting, - kReplicaPrivateKeyForTesting, - kPublicKeysOfReplicasForTesting, - kKeyFormatForTesting, - &kInitialPublicKeysOfClientsForTesting, - kKeyFormatForTesting, - *sigManagerReplicasInfoForTesting)); + sig_manager_for_key_exchange_manager = SigManager::init(kReplicaIdForTesting, + kReplicaPrivateKeyForTesting, + kPublicKeysOfReplicasForTesting, + kKeyFormatForTesting, + &kInitialPublicKeysOfClientsForTesting, + kKeyFormatForTesting, + *sigManagerReplicasInfoForTesting); } static void initializeKeyExchangeManagerForClientsManagerTesting() { diff --git a/bftengine/tests/keyManager/KeyManager_test.cpp b/bftengine/tests/keyManager/KeyManager_test.cpp index 1fa6fc988d..f5144cb380 100644 --- a/bftengine/tests/keyManager/KeyManager_test.cpp +++ b/bftengine/tests/keyManager/KeyManager_test.cpp @@ -33,13 +33,16 @@ struct DummyKeyGen : public IMultiSigKeyGenerator, public IKeyExchanger { std::tuple generateMultisigKeyPair() { return std::make_tuple(prv, pub, concord::crypto::SignatureAlgorithm::Uninitialized); } + void onPrivateKeyExchange(const std::string& secretKey, const std::string& verificationKey) { selfpub = verificationKey; selfprv = secretKey; } + void onPublicKeyExchange(const std::string& verificationKey, const std::uint16_t& signerIndex) { pubs[signerIndex] = verificationKey; } + DummyKeyGen(uint32_t cs) : pubs{cs, ""} {} std::string prv; std::string pub; diff --git a/bftengine/tests/messages/CheckpointMsg_test.cpp b/bftengine/tests/messages/CheckpointMsg_test.cpp index bd46d62eef..99fe6d16d8 100644 --- a/bftengine/tests/messages/CheckpointMsg_test.cpp +++ b/bftengine/tests/messages/CheckpointMsg_test.cpp @@ -39,7 +39,7 @@ class CheckpointMsgTestsFixture : public ::testing::Test { {} ReplicaConfig& config; ReplicasInfo replicaInfo; - std::unique_ptr sigManager; + std::shared_ptr sigManager; static const char rawSpanContext[]; static const std::string spanContext; diff --git a/bftengine/tests/messages/ClientRequestMsg_test.cpp b/bftengine/tests/messages/ClientRequestMsg_test.cpp index a33c499eaa..7ed6843087 100644 --- a/bftengine/tests/messages/ClientRequestMsg_test.cpp +++ b/bftengine/tests/messages/ClientRequestMsg_test.cpp @@ -37,7 +37,7 @@ class ClientRequestMsgTestFixture : public ::testing::Test { ReplicaConfig& config; ReplicasInfo replicaInfo; - std::unique_ptr sigManager; + std::shared_ptr sigManager; }; TEST_F(ClientRequestMsgTestFixture, create_and_compare) { diff --git a/bftengine/tests/messages/PrePrepareMsg_test.cpp b/bftengine/tests/messages/PrePrepareMsg_test.cpp index aad58df33e..f7277c0024 100644 --- a/bftengine/tests/messages/PrePrepareMsg_test.cpp +++ b/bftengine/tests/messages/PrePrepareMsg_test.cpp @@ -106,7 +106,7 @@ class PrePrepareMsgTestFixture : public ::testing::Test { ReplicaConfig& config; ReplicasInfo replicaInfo; - std::unique_ptr sigManager; + std::shared_ptr sigManager; }; TEST_F(PrePrepareMsgTestFixture, finalize_and_validate) { diff --git a/bftengine/tests/messages/ReplicaAsksToLeaveViewMsg_test.cpp b/bftengine/tests/messages/ReplicaAsksToLeaveViewMsg_test.cpp index fa1261d468..6825977398 100644 --- a/bftengine/tests/messages/ReplicaAsksToLeaveViewMsg_test.cpp +++ b/bftengine/tests/messages/ReplicaAsksToLeaveViewMsg_test.cpp @@ -31,7 +31,7 @@ TEST(ReplicaAsksToLeaveViewMsg, base_methods) { const char rawSpanContext[] = {"span_\0context"}; const std::string spanContext{rawSpanContext, sizeof(rawSpanContext)}; ReplicasInfo replicaInfo(config, true, true); - std::unique_ptr sigManager(createSigManager(config.replicaId, + std::shared_ptr sigManager(createSigManager(config.replicaId, config.replicaPrivateKey, concord::crypto::KeyFormat::HexaDecimalStrippedFormat, config.publicKeysOfReplicas, diff --git a/bftengine/tests/messages/ReplicaRestartReadyMsg_test.cpp b/bftengine/tests/messages/ReplicaRestartReadyMsg_test.cpp index 699600b24f..fc4aea0f4d 100644 --- a/bftengine/tests/messages/ReplicaRestartReadyMsg_test.cpp +++ b/bftengine/tests/messages/ReplicaRestartReadyMsg_test.cpp @@ -34,7 +34,7 @@ TEST(ReplicaRestartReadyMsg, base_methods) { const char rawSpanContext[] = {"span_\0context"}; const std::string spanContext{rawSpanContext, sizeof(rawSpanContext)}; ReplicasInfo replicaInfo(config, true, true); - std::unique_ptr sigManager(createSigManager(config.replicaId, + std::shared_ptr sigManager(createSigManager(config.replicaId, config.replicaPrivateKey, concord::crypto::KeyFormat::HexaDecimalStrippedFormat, config.publicKeysOfReplicas, @@ -58,7 +58,7 @@ TEST(ReplicaRestartReadyMsg, with_extraData) { const std::string spanContext{rawSpanContext, sizeof(rawSpanContext)}; ReplicasInfo replicaInfo(config, true, true); auto version = "latest"; - std::unique_ptr sigManager(createSigManager(config.replicaId, + std::shared_ptr sigManager(createSigManager(config.replicaId, config.replicaPrivateKey, concord::crypto::KeyFormat::HexaDecimalStrippedFormat, config.publicKeysOfReplicas, diff --git a/bftengine/tests/messages/ViewChangeMsg_test.cpp b/bftengine/tests/messages/ViewChangeMsg_test.cpp index 334958b7ee..6730804f93 100644 --- a/bftengine/tests/messages/ViewChangeMsg_test.cpp +++ b/bftengine/tests/messages/ViewChangeMsg_test.cpp @@ -47,7 +47,7 @@ void ViewChangeMsgTestsFixture::ViewChangeMsgTests(bool bAddElements, ViewNum viewNum = 2u; SeqNum seqNum = 3u; ReplicasInfo replicaInfo(config, true, true); - std::unique_ptr sigManager(createSigManager(config.replicaId, + std::shared_ptr sigManager(createSigManager(config.replicaId, config.replicaPrivateKey, concord::crypto::KeyFormat::HexaDecimalStrippedFormat, config.publicKeysOfReplicas, @@ -191,7 +191,7 @@ void ViewChangeMsgTestsFixture::ViewChangeMsgAddRemoveComplaints(const std::stri ViewNum viewNum = 2u; SeqNum seqNum = 3u; ReplicasInfo replicaInfo(config, true, true); - std::unique_ptr sigManager(createSigManager(config.replicaId, + std::shared_ptr sigManager(createSigManager(config.replicaId, config.replicaPrivateKey, concord::crypto::KeyFormat::HexaDecimalStrippedFormat, config.publicKeysOfReplicas, diff --git a/bftengine/tests/messages/helper.cpp b/bftengine/tests/messages/helper.cpp index 0206dc0b2f..0e01a16c40 100644 --- a/bftengine/tests/messages/helper.cpp +++ b/bftengine/tests/messages/helper.cpp @@ -17,60 +17,20 @@ typedef std::pair IdToKeyPair; using concord::crypto::SignatureAlgorithm; using bftEngine::ReplicaConfig; -const std::string replicaRSAPrivateKey = { - "308204BC020100300D06092A864886F70D0101010500048204A6308204A20201000282010100BCC5BEA607F4F52A493AA2F40C2D5482D7CE37" - "DFC526E98131FDC92CE2ECA6035DB307B182EF52CA8471B78A65E445399816AFACB224F4CEA9597D4B6FE5E84030B7AF78A88BA0233263A9F0" - "E2658A6E5BE57923D9093B7D6B70FDBAEC3CDA05C5EDE237674A598F5D607A50C1C528EEAE4B690C90820901A01BF4747C39FE6BD6DA535A9B" - "8CAE04B39D5D5158C8FFD2CD6652195AC48646B8AEF202615306575FD9508632E6027ABD50E786BD1A984C7DD11E36293A45EDBBFB61E438C1" - "89C2B73A69F6605C909F98B6C3F795354BBB988C9695F8A1E27FFC3CE4FFA64B549DD9072763404FBD352C5C1A05FA3D17377E113600B1EDCA" - "EE17687BC4C1AA6F3D020111028201000A2DCCCA35228AB5BB2A1050EC94034D546BBF3F845B323CED5CBE4C75A5DBC674BD1AC48D55B062C3" - "607C17C6BFC27A523D7564EB7CF91F38D15FDAA7EA83BD2FCDAB581320A07A5E532D8E3E675B8AF867FB3CE9D21102FF84D6774171B66C3B9D" - "240A84508EDE51958415EC54AB9E704CD9BE2AEDE9E5BC1595F738E5026C9FB204D15017B72A7C22A9026BEC239EF9B329BBA56384D625217C" - "5C9513EFD10A4DC84B37505C99EBD2695F3CE59ABECD280FA7014865F0FD6E48F34AB0ADC2F66DF738A3C058818BB321DDF0D7AB17EACEE6B5" - "5AADD2C0D9522FB6E001D945C014DA857FE347FF41F29DFCA5E9BA9779B26224F65503AD7FE6AF92A63C1EB902818100F16B929438481F3EC0" - "64FC91AD9F41CB95368C06221FE323E02547FBECA9A65CE7EC3C70C2F35D48BEB0A70F9CDF5D367962AD6F80438A758F1E84425D168E3BF16B" - "6FD9F73B3A77ECA80CA1845C0C89BA46DD6D9234621A713803D0DC872FBE4C6E372D04FB28454978C319D187FAEFBE78AE3E2D4D36DEC11869" - "2C9C56162902818100C82C38E2675E329AC6870B1E655440B17E7AEEA9C911EC599C4B335179662F0444791D2E284ED89717BF29EDFCB0625A" - "CA9EC2DC05898777453CAC01E6DE650A02EEE2BC2F323296A488FD709E50DA307E8E76EDD4BE98F8B56DC963A6B921A373D0617326235DB7D6" - "0A13876997F4F17A4F3D150960ACF2125414B2A610AAF502818100AA6A0D1D54E79D95B4FBFD94021610537862BD31817FEBA0DA74AB486AD2" - "1B14677994135C6F8D244A5E940B05525FEA3790F2E54B7AF852FB9D1210BA2E0A0C31F17C216338DDFA4CB2DBBD4E5F17E8BFB98D3E4915EA" - "E57D187B2A051421B3813EBD8930B1499A51FAE412398D299A2C18F3772F0953E8884D776AC8B53CD10281803ADFD47ED31BB487E00999DBC3" - "73221616242813D1B9FA3879434B5432C3B379B9C944D157263FB3F7ECEE36EFF7A4750E6AEE047A196414054E147907AAD26C5B3733A0C296" - "4B1D3F7395D5D435E5D2071AD7AF5CB08758355C8686B890CDA88B798612CEFB57CCA85D5109B5A529ECAB80B79CC685D8836ECD6F7FD67D5F" - "7502818100B33DC57C801E0824CF2C77D6D35EC51E321168DA1DED72238ECF69DF6BD485B19A2A67CFBE87F6819F5872463687295F4091C6D9" - "9AE98AD08EB45931E761D42D9CE941CEF7DF8A493FEAD8EB571BBBA21EF6403151CB25C71A9BB457D3FB058AA34AB4C1AB474C86293A26D428" - "E77960457E2631215FF7B68013877ABCCE4322"}; -const std::string replicaRSAPubKey = { - "30820120300D06092A864886F70D01010105000382010D00308201080282010100B" - "CC5BEA607F4F52A493AA2F40C2D5482D7CE37DFC526E981" - "31FDC92CE2ECA6035DB307B182EF52CA8471B78A65E445399816AFACB224F4CEA95" - "97D4B6FE5E84030B7AF78A88BA0233263A9F0E2658A6E5B" - "E57923D9093B7D6B70FDBAEC3CDA05C5EDE237674A598F5D607A50C1C528EEAE4B6" - "90C90820901A01BF4747C39FE6BD6DA535A9B8CAE04B39D" - "5D5158C8FFD2CD6652195AC48646B8AEF202615306575FD9508632E6027ABD50E78" - "6BD1A984C7DD11E36293A45EDBBFB61E438C189C2B73A69" - "F6605C909F98B6C3F795354BBB988C9695F8A1E27FFC3CE4FFA64B549DD90727634" - "04FBD352C5C1A05FA3D17377E113600B1EDCAEE17687BC4" - "C1AA6F3D020111"}; const std::string replicaEdDSAPrivateKey = {"09a30490ebf6f6685556046f2497fd9c7df4a552998c9a9b6ebec742e8183174"}; const std::string replicaEdDSAPubKey = {"7363bc5ab96d7f85e71a5ffe0b284405ae38e2e0f032fb3ffe805d9f0e2d117b"}; void loadPrivateAndPublicKeys(std::string& myPrivateKey, std::set>& publicKeysOfReplicas, - ReplicaId myId, size_t numReplicas) { ConcordAssert(numReplicas <= 7); std::string pubKey; - if (ReplicaConfig::instance().replicaMsgSigningAlgo == SignatureAlgorithm::EdDSA) { - myPrivateKey = replicaEdDSAPrivateKey; - pubKey = replicaEdDSAPubKey; - } - const std::vector replicasPubKeys{pubKey, pubKey, pubKey, pubKey, pubKey, pubKey, pubKey}; + ConcordAssertEQ(ReplicaConfig::instance().replicaMsgSigningAlgo, SignatureAlgorithm::EdDSA); + myPrivateKey = replicaEdDSAPrivateKey; for (size_t i{0}; i < numReplicas; ++i) { - if (i == myId) continue; - publicKeysOfReplicas.insert(IdToKeyPair(i, replicasPubKeys[i].c_str())); + publicKeysOfReplicas.insert(IdToKeyPair(i, replicaEdDSAPubKey.c_str())); } } @@ -95,23 +55,71 @@ bftEngine::ReplicaConfig& createReplicaConfig(uint16_t fVal, uint16_t cVal) { config.threadbagConcurrencyLevel1 = 16; config.threadbagConcurrencyLevel2 = 8; - loadPrivateAndPublicKeys(config.replicaPrivateKey, config.publicKeysOfReplicas, config.replicaId, config.numReplicas); + loadPrivateAndPublicKeys(config.replicaPrivateKey, config.publicKeysOfReplicas, config.numReplicas); - bftEngine::CryptoManager::instance(std::make_unique()); + bftEngine::CryptoManager::init(std::make_unique()); return config; } -bftEngine::impl::SigManager* createSigManager(size_t myId, +TestMultisigCryptoSystem::TestMultisigCryptoSystem(NodeIdType id, + const std::vector& hexStringEdDSAPublicKeys, + const std::string& hexStringEdDSAPrivateKey) + : id_{id}, + hexStringEdDSAPrivateKey_{hexStringEdDSAPrivateKey}, + hexStringEdDSAPublicKeys_{hexStringEdDSAPublicKeys} {} +IThresholdVerifier* TestMultisigCryptoSystem::createThresholdVerifier(uint16_t threshold) { + return factory_.newVerifier(threshold, hexStringEdDSAPublicKeys_.size(), "", hexStringEdDSAPublicKeys_); +} +IThresholdSigner* TestMultisigCryptoSystem::createThresholdSigner() { + return factory_.newSigner(id_, hexStringEdDSAPrivateKey_.c_str()); +} + +class TestSigManager : public bftEngine::impl::SigManager { + public: + TestSigManager(PrincipalId myId, + const std::pair& mySigPrivateKey, + const std::vector>& publickeys, + const std::map& publicKeysMapping, + const ReplicasInfo& replicasInfo, + bool transactionSigningEnabled = false) + : bftEngine::impl::SigManager( + myId, mySigPrivateKey, publickeys, publicKeysMapping, transactionSigningEnabled, replicasInfo) {} + + static std::shared_ptr init(size_t myId, std::string& myPrivateKey, - concord::crypto::KeyFormat replicasKeysFormat, - std::set>& publicKeysOfReplicas, + std::set>& publicKeys, ReplicasInfo& replicasInfo) { - return SigManager::init(myId, - myPrivateKey, - publicKeysOfReplicas, - replicasKeysFormat, - nullptr, - concord::crypto::KeyFormat::PemFormat, - replicasInfo); + std::vector> publicKeysWithFormat(publicKeys.size()); + std::vector replicaPublicKeys(replicasInfo.getNumberOfReplicas()); + + std::map publicKeysMapping; + for (auto& [id, key] : publicKeys) { + publicKeysWithFormat[id] = {key, concord::crypto::KeyFormat::HexaDecimalStrippedFormat}; + publicKeysMapping.emplace(id, id); + ConcordAssert(replicasInfo.isIdOfReplica(id)); + replicaPublicKeys[id] = key; + } + + bftEngine::CryptoManager::init(std::make_unique(myId, replicaPublicKeys, myPrivateKey)); + auto manager = + std::make_shared(myId, + std::pair{ + myPrivateKey, concord::crypto::KeyFormat::HexaDecimalStrippedFormat}, + publicKeysWithFormat, + publicKeysMapping, + replicasInfo); + SigManager::reset(manager); + return manager; + } +}; + +std::shared_ptr createSigManager( + size_t myId, + std::string& myPrivateKey, + concord::crypto::KeyFormat replicasKeysFormat, + std::set>& publicKeysOfReplicas, + ReplicasInfo& replicasInfo) { + UNUSED(replicasKeysFormat); + return TestSigManager::init(myId, myPrivateKey, publicKeysOfReplicas, replicasInfo); } diff --git a/bftengine/tests/messages/helper.hpp b/bftengine/tests/messages/helper.hpp index 172fe38eca..48eefcc25a 100644 --- a/bftengine/tests/messages/helper.hpp +++ b/bftengine/tests/messages/helper.hpp @@ -26,6 +26,7 @@ #include "crypto/threshsign/IPublicKey.h" #include "CryptoManager.hpp" #include "SigManager.hpp" +#include "threshsign/eddsa/EdDSAMultisigFactory.h" using bftEngine::impl::ReplicasInfo; @@ -39,9 +40,14 @@ class IShareVerificationKeyDummy : public IShareVerificationKey { std::string toString() const override { return "IShareVerificationKeyDummy"; } }; -class IThresholdSignerDummy : public IThresholdSigner { +class IThresholdSignerDummy : public IThresholdSigner, concord::crypto::ISigner { public: int requiredLengthForSignedData() const override { return 2048; } + size_t signBuffer(const concord::Byte *dataIn, size_t dataLen, concord::Byte *sigOutBuffer) const override { + return 0; + } + size_t signatureLength() const override { return requiredLengthForSignedData(); }; + std::string getPrivKey() const override { return shareSecretKey.toString(); }; void signData(const char *hash, int hashLen, char *outSig, int outSigLen) override { std::memset(outSig, 'S', outSigLen); } @@ -114,13 +120,28 @@ void testMessageBaseMethods(const MessageT &tested, MsgType type, NodeIdType sen EXPECT_TRUE(other->equals(*deserialized)); } -bftEngine::impl::SigManager *createSigManager(size_t myId, - std::string &myPrivateKey, - concord::crypto::KeyFormat replicasKeysFormat, - std::set> &publicKeysOfReplicas, - ReplicasInfo &replicasInfo); +std::shared_ptr createSigManager( + size_t myId, + std::string &myPrivateKey, + concord::crypto::KeyFormat replicasKeysFormat, + std::set> &publicKeysOfReplicas, + ReplicasInfo &replicasInfo); void loadPrivateAndPublicKeys(std::string &myPrivateKey, std::set> &publicKeysOfReplicas, - ReplicaId myId, size_t numReplicas); + +class TestMultisigCryptoSystem : public Cryptosystem { + public: + TestMultisigCryptoSystem(NodeIdType id, + const std::vector &hexStringEdDSAPublicKeys, + const std::string &hexStringEdDSAPrivateKey); + IThresholdVerifier *createThresholdVerifier(uint16_t threshold = 0) override; + IThresholdSigner *createThresholdSigner() override; + + private: + NodeIdType id_; + std::string hexStringEdDSAPrivateKey_; + std::vector hexStringEdDSAPublicKeys_; + EdDSAMultisigFactory factory_; +}; \ No newline at end of file diff --git a/bftengine/tests/testMsgsCertificate/msgsCertificate_test.cpp b/bftengine/tests/testMsgsCertificate/msgsCertificate_test.cpp index 4c4bb63066..7f9f770db3 100644 --- a/bftengine/tests/testMsgsCertificate/msgsCertificate_test.cpp +++ b/bftengine/tests/testMsgsCertificate/msgsCertificate_test.cpp @@ -38,7 +38,7 @@ class msgsCertificateTestsFixture : public ::testing::Test { {} ReplicaConfig& config; ReplicasInfo replicaInfo; - std::unique_ptr sigManager; + std::shared_ptr sigManager; void test_state_still_consistent() { MsgsCertificate replysCertificate( diff --git a/bftengine/tests/testSerialization/TestSerialization.cpp b/bftengine/tests/testSerialization/TestSerialization.cpp index 623859a9ef..7bed8c3318 100644 --- a/bftengine/tests/testSerialization/TestSerialization.cpp +++ b/bftengine/tests/testSerialization/TestSerialization.cpp @@ -393,7 +393,7 @@ void testCheckDescriptorOfLastStableCheckpoint(bool init) { int main() { auto &config = createReplicaConfig(); ReplicasInfo replicaInfo(config, false, false); - std::unique_ptr sigManager(createSigManager(config.replicaId, + std::shared_ptr sigManager(createSigManager(config.replicaId, config.replicaPrivateKey, concord::crypto::KeyFormat::HexaDecimalStrippedFormat, config.publicKeysOfReplicas, diff --git a/bftengine/tests/testViewChange/testViewChange.cpp b/bftengine/tests/testViewChange/testViewChange.cpp index 6c0f709aab..c4e7dbfcd9 100644 --- a/bftengine/tests/testViewChange/testViewChange.cpp +++ b/bftengine/tests/testViewChange/testViewChange.cpp @@ -50,7 +50,7 @@ static const int F = 1; static const int C = 0; ViewChangeSafetyLogic::Restriction restrictions[kWorkWindowSize]; std::unique_ptr pRepInfo; -std::unique_ptr sigManager_; +std::shared_ptr sigManager_; class DummyShareSecretKey : public IShareSecretKey { public: @@ -82,14 +82,14 @@ void setUpConfiguration_4() { replicaConfig[i].cVal = C; replicaConfig[i].replicaId = i; } - loadPrivateAndPublicKeys(replicaConfig[0].replicaPrivateKey, replicaConfig[0].publicKeysOfReplicas, 0, N); + loadPrivateAndPublicKeys(replicaConfig[0].replicaPrivateKey, replicaConfig[0].publicKeysOfReplicas, N); pRepInfo = std::make_unique(replicaConfig[0], true, true); - sigManager_.reset(createSigManager(0, - replicaConfig[0].replicaPrivateKey, - concord::crypto::KeyFormat::HexaDecimalStrippedFormat, - replicaConfig[0].publicKeysOfReplicas, - *pRepInfo)); + sigManager_ = createSigManager(0, + replicaConfig[0].replicaPrivateKey, + concord::crypto::KeyFormat::HexaDecimalStrippedFormat, + replicaConfig[0].publicKeysOfReplicas, + *pRepInfo); } TEST(testViewchangeSafetyLogic_test, computeRestrictions) { @@ -482,7 +482,7 @@ TEST(testViewchangeSafetyLogic_test, empty_correct_VC_msgs) { int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); setUpConfiguration_4(); - bftEngine::CryptoManager::instance(std::make_unique()); + bftEngine::CryptoManager::init(std::make_unique()); int res = RUN_ALL_TESTS(); // TODO cleanup the generated certificates return res; diff --git a/bftengine/tests/testViewChange/test_views_manager.cpp b/bftengine/tests/testViewChange/test_views_manager.cpp index a66ef5bbf2..95e9a85fb8 100644 --- a/bftengine/tests/testViewChange/test_views_manager.cpp +++ b/bftengine/tests/testViewChange/test_views_manager.cpp @@ -48,7 +48,7 @@ class ViewsManagerTest : public ::testing::Test { protected: ReplicaConfig& rc; ReplicasInfo replicasInfo; - std::unique_ptr sigManager; + std::shared_ptr sigManager; std::unique_ptr viewsManager; ViewsManagerTest() @@ -59,11 +59,13 @@ class ViewsManagerTest : public ::testing::Test { concord::crypto::KeyFormat::HexaDecimalStrippedFormat, rc.publicKeysOfReplicas, replicasInfo)), - viewsManager(std::make_unique(&replicasInfo)) {} + viewsManager(std::make_unique(&replicasInfo)) { + bftEngine::ReservedPagesClientBase::setReservedPages(&res_pages_mock_); + CryptoManager::init(std::make_unique()); + } }; TEST_F(ViewsManagerTest, moving_to_higher_view) { - bftEngine::ReservedPagesClientBase::setReservedPages(&res_pages_mock_); const auto currentView = viewsManager->getCurrentView(); LOG_INFO(GL, KVLOG(currentView)); viewsManager->setHigherView(currentView + 1); @@ -72,7 +74,6 @@ TEST_F(ViewsManagerTest, moving_to_higher_view) { } TEST_F(ViewsManagerTest, store_complaint) { - bftEngine::ReservedPagesClientBase::setReservedPages(&res_pages_mock_); std::unique_ptr complaint(ReplicaAsksToLeaveViewMsg::create( rc.replicaId, initialView, ReplicaAsksToLeaveViewMsg::Reason::ClientRequestTimeout)); @@ -84,7 +85,6 @@ TEST_F(ViewsManagerTest, store_complaint) { } TEST_F(ViewsManagerTest, form_a_quorum_of_complaints) { - bftEngine::ReservedPagesClientBase::setReservedPages(&res_pages_mock_); for (int replicaId = 0; replicaId < rc.numReplicas; ++replicaId) { if (replicaId == rc.replicaId) continue; @@ -100,7 +100,6 @@ TEST_F(ViewsManagerTest, form_a_quorum_of_complaints) { } TEST_F(ViewsManagerTest, status_message_with_complaints) { - bftEngine::ReservedPagesClientBase::setReservedPages(&res_pages_mock_); bftEngine::impl::SeqNum lastExecutedSeqNum = 200; const bool viewIsActive = true; const bool hasNewChangeMsg = true; @@ -134,7 +133,6 @@ TEST_F(ViewsManagerTest, status_message_with_complaints) { } TEST_F(ViewsManagerTest, get_quorum_for_next_view_on_view_change_message_with_enough_complaints) { - bftEngine::ReservedPagesClientBase::setReservedPages(&res_pages_mock_); bftEngine::impl::ReplicaId sourceReplicaId = (rc.replicaId + 1) % rc.numReplicas; std::set otherReplicas; const int nextView = initialView + 1; @@ -163,7 +161,6 @@ TEST_F(ViewsManagerTest, get_quorum_for_next_view_on_view_change_message_with_en } TEST_F(ViewsManagerTest, get_quorum_for_higher_view_on_view_change_message_with_enough_complaints) { - bftEngine::ReservedPagesClientBase::setReservedPages(&res_pages_mock_); bftEngine::impl::ReplicaId sourceReplicaId = (rc.replicaId + 1) % rc.numReplicas; std::set otherReplicas; const int higherView = initialView + 2; @@ -192,7 +189,6 @@ TEST_F(ViewsManagerTest, get_quorum_for_higher_view_on_view_change_message_with_ } TEST_F(ViewsManagerTest, adding_view_change_messages_to_status_message) { - bftEngine::ReservedPagesClientBase::setReservedPages(&res_pages_mock_); const bftEngine::impl::ReplicaId firstMessageSourceReplicaId = (rc.replicaId + 1) % rc.numReplicas, secondMessageSourceReplicaId = (rc.replicaId + 2) % rc.numReplicas; const int nextView = initialView + 1; @@ -262,11 +258,11 @@ TEST_F(ViewsManagerTest, adding_pre_prepare_messages_to_status_message) { prePrepareMsg->finishAddingRequests(); const auto primary = replicasInfo.primaryOfView(initialView); - char buff[32]{}; - + std::vector buff(Digest::size()); // Create a PrepareFullMessage. When this message is added as an element to a view change message, // it will be used to create a PreparedCertificate. - PrepareFullMsg* prepareFullMsg = PrepareFullMsg::create(initialView, lastExecutedSeqNum, primary, buff, sizeof(buff)); + PrepareFullMsg* prepareFullMsg = + PrepareFullMsg::create(initialView, lastExecutedSeqNum, primary, buff.data(), buff.size()); const int N = rc.numReplicas; ViewChangeMsg** viewChangeMsgs = new ViewChangeMsg*[N]; @@ -346,7 +342,6 @@ TEST_F(ViewsManagerTest, adding_pre_prepare_messages_to_status_message) { } TEST_F(ViewsManagerTest, trigger_view_change) { - bftEngine::ReservedPagesClientBase::setReservedPages(&res_pages_mock_); ASSERT_EQ(rc.numReplicas, 3 * numberOfFaultyReplicas + 1); ASSERT_EQ(rc.fVal, numberOfFaultyReplicas); ASSERT_EQ(rc.cVal, 0); diff --git a/client/bftclient/src/bft_client.cpp b/client/bftclient/src/bft_client.cpp index 806157b66e..3c8000f391 100644 --- a/client/bftclient/src/bft_client.cpp +++ b/client/bftclient/src/bft_client.cpp @@ -29,6 +29,8 @@ using bftEngine::ReplicaConfig; namespace bft::client { +static constexpr size_t replicaIdentityHistoryCount = 2; + Client::Client(SharedCommPtr comm, const ClientConfig& config, std::shared_ptr aggregator) : communication_(std::move(comm)), config_(config), @@ -70,9 +72,11 @@ Client::Client(SharedCommPtr comm, const ClientConfig& config, std::shared_ptrstart(); if (config_.replicas_master_key_folder_path.has_value()) { bft::communication::StateControl::instance().setGetPeerPubKeyMethod([&](uint32_t rid) { + // The client reads a public key file which the cre client which runs on the same machine maintains + // the cre client updates the file after polling updates about replica key exchanges concord::secretsmanager::SecretsManagerPlain psm_; std::string key_path = config_.replicas_master_key_folder_path.value() + "/" + std::to_string(rid) + "/pub_key"; - return psm_.decryptFile(key_path).value_or(""); + return std::array{psm_.decryptFile(key_path).value_or(""), ""}; }); } } diff --git a/client/reconfiguration/src/default_handlers.cpp b/client/reconfiguration/src/default_handlers.cpp index a33f5b92bf..b8b161161f 100644 --- a/client/reconfiguration/src/default_handlers.cpp +++ b/client/reconfiguration/src/default_handlers.cpp @@ -44,6 +44,7 @@ bool ReplicaMainKeyPublicationHandler::validate(const State& state) const { return validateInputState(state); } bool ReplicaMainKeyPublicationHandler::execute(const State& state, WriteState&) { + LOG_INFO(getLogger(), "In ReplicaMainKeyPublicationHandler"); auto cmd = getCmdFromInputState(state); fs::path path = fs::path(output_dir_) / std::to_string(cmd.sender_id); auto curr_key = file_handler_.decryptFile((path / "pub_key").string()).value_or(""); diff --git a/client/reconfiguration/src/poll_based_state_client.cpp b/client/reconfiguration/src/poll_based_state_client.cpp index 535593f2c0..d0cb6f39de 100644 --- a/client/reconfiguration/src/poll_based_state_client.cpp +++ b/client/reconfiguration/src/poll_based_state_client.cpp @@ -70,6 +70,7 @@ std::vector PollBasedStateClient::getStateUpdate(bool& succ) const { rreq.sender = id_; rreq.command = creq; auto sn = sn_gen_.unique(); + LOG_INFO(getLogger(), "sending reconfig request" << KVLOG(sn)); auto rres = sendReconfigurationRequest(rreq, "getStateUpdate-" + std::to_string(sn), sn, true); if (!rres.success) { LOG_WARN(getLogger(), "invalid response from replicas"); @@ -112,6 +113,7 @@ PollBasedStateClient::~PollBasedStateClient() { void PollBasedStateClient::start() { stopped = false; consumer_ = std::thread([&]() { + LOG_INFO(getLogger(), "Client started" << KVLOG(interval_timeout_ms_)); while (!stopped) { { std::unique_lock lk(resume_lock_); @@ -120,10 +122,12 @@ void PollBasedStateClient::start() { } } + LOG_INFO(getLogger(), "Sleeping for" << KVLOG(interval_timeout_ms_)); std::this_thread::sleep_for(std::chrono::milliseconds(interval_timeout_ms_)); - if (stopped) return; + if (stopped) break; std::lock_guard lk(lock_); bool succ; + LOG_INFO(getLogger(), "Getting state update"); auto new_state = getStateUpdate(succ); uint64_t max_update_block{0}; for (const auto& s : new_state) { @@ -135,6 +139,7 @@ void PollBasedStateClient::start() { new_updates_.notify_one(); if (max_update_block > last_known_block_) last_known_block_ = max_update_block; } + LOG_INFO(getLogger(), "Client stopped"); }); } void PollBasedStateClient::stop() { @@ -161,10 +166,12 @@ bool PollBasedStateClient::updateState(const WriteState& state) { return rres.success; } void PollBasedStateClient::halt() { + LOG_INFO(getLogger(), "Halting client"); std::unique_lock lk(resume_lock_); halted_ = true; } void PollBasedStateClient::resume() { + LOG_INFO(getLogger(), "Resuming client"); std::unique_lock lk(resume_lock_); halted_ = false; resume_cond_.notify_one(); diff --git a/kvbc/include/categorization/blockchain.h b/kvbc/include/categorization/blockchain.h index c617f1dac8..5f45cfb9c6 100644 --- a/kvbc/include/categorization/blockchain.h +++ b/kvbc/include/categorization/blockchain.h @@ -49,6 +49,8 @@ class Blockchain { } void addBlock(const Block& new_block, storage::rocksdb::NativeWriteBatch& wb) { + LOG_INFO(GL, "Adding block_id: " << new_block.id() << " to the database"); + printCallStack(); wb.put(detail::BLOCKS_CF, Block::generateKey(new_block.id()), Block::serialize(new_block)); } diff --git a/kvbc/include/reconfiguration_kvbc_handler.hpp b/kvbc/include/reconfiguration_kvbc_handler.hpp index 94bce96728..d97ac756e7 100644 --- a/kvbc/include/reconfiguration_kvbc_handler.hpp +++ b/kvbc/include/reconfiguration_kvbc_handler.hpp @@ -73,7 +73,7 @@ class StateSnapshotReconfigurationHandler : public ReconfigurationBlockTools, concord::messages::ReconfigurationResponse&) override; bool handle(const concord::messages::SignedPublicStateHashRequest&, - uint64_t, + uint64_t bft_seq_num, uint32_t, const std::optional&, concord::messages::ReconfigurationResponse&) override; @@ -230,7 +230,7 @@ class ReconfigurationHandler : public concord::reconfiguration::OperatorCommands concord::messages::ReconfigurationResponse&) override; bool handle(const concord::messages::UnwedgeCommand&, - uint64_t, + uint64_t bft_seq_num, uint32_t, const std::optional&, concord::messages::ReconfigurationResponse&) override; @@ -314,7 +314,8 @@ class InternalPostKvReconfigurationHandler : public concord::reconfiguration::IR InternalPostKvReconfigurationHandler(kvbc::IBlockAdder& block_adder, kvbc::IReader& ro_storage) : ReconfigurationBlockTools{block_adder, ro_storage} {} bool verifySignature(uint32_t sender_id, const std::string& data, const std::string& signature) const override { - return true; + if (!bftEngine::impl::SigManager::instance()->hasVerifier(sender_id)) return false; + return bftEngine::impl::SigManager::instance()->verifySig(sender_id, data, signature); } bool handle(const concord::messages::ClientExchangePublicKey& command, diff --git a/kvbc/src/reconfiguration_kvbc_handler.cpp b/kvbc/src/reconfiguration_kvbc_handler.cpp index 7e2a284cd8..8606fe0f68 100644 --- a/kvbc/src/reconfiguration_kvbc_handler.cpp +++ b/kvbc/src/reconfiguration_kvbc_handler.cpp @@ -280,7 +280,7 @@ bool StateSnapshotReconfigurationHandler::handle(const concord::messages::StateS } bool StateSnapshotReconfigurationHandler::handle(const concord::messages::SignedPublicStateHashRequest& req, - uint64_t, + uint64_t bft_seq_num, uint32_t, const std::optional&, concord::messages::ReconfigurationResponse& reconf_resp) { @@ -325,7 +325,8 @@ bool StateSnapshotReconfigurationHandler::handle(const concord::messages::Signed resp.data.hash = public_state_hash.hash; resp.signature.assign(SigManager::instance()->getMySigLength(), 0); const auto data_ser = serialize(resp.data); - SigManager::instance()->sign(reinterpret_cast(data_ser.data()), + SigManager::instance()->sign(bft_seq_num, + reinterpret_cast(data_ser.data()), data_ser.size(), reinterpret_cast(resp.signature.data())); LOG_INFO(getLogger(), @@ -422,6 +423,8 @@ concord::messages::ClientStateReply KvbcClientReconfigurationHandler::buildRepli creply.block_id = 0; auto res = ro_storage_.getLatest(concord::kvbc::categorization::kConcordReconfigurationCategoryId, command_type + std::to_string(clientid)); + LOG_INFO(GL, "Building state reply" << KVLOG(clientid, command_type, res.has_value())); + printCallStack(); if (res.has_value()) { std::visit( [&](auto&& arg) { @@ -465,6 +468,7 @@ bool KvbcClientReconfigurationHandler::handle(const concord::messages::ClientRec concord::messages::ReconfigurationResponse& rres) { concord::messages::ClientReconfigurationStateReply rep; uint16_t first_client_id = ReplicaConfig::instance().numReplicas + ReplicaConfig::instance().numRoReplicas; + LOG_INFO(GL, KVLOG(first_client_id, sender_id)); if (sender_id > first_client_id) { for (uint8_t i = kvbc::keyTypes::CLIENT_COMMAND_TYPES::start_ + 1; i < kvbc::keyTypes::CLIENT_COMMAND_TYPES::end_; i++) { @@ -477,23 +481,28 @@ bool KvbcClientReconfigurationHandler::handle(const concord::messages::ClientRec if (ke_csrep.block_id > 0) rep.states.push_back(ke_csrep); } } else { - auto scaling_key_prefix = + static const std::vector non_external_client_update_types = { + // TLS key exchange update + std::string{kvbc::keyTypes::reconfiguration_tls_exchange_key}, + // Scaling command + std::string{kvbc::keyTypes::reconfiguration_client_data_prefix, + static_cast(kvbc::keyTypes::CLIENT_COMMAND_TYPES::CLIENT_SCALING_EXECUTE_COMMAND)}, + // Scaling status update std::string{kvbc::keyTypes::reconfiguration_client_data_prefix, - static_cast(kvbc::keyTypes::CLIENT_COMMAND_TYPES::CLIENT_SCALING_EXECUTE_COMMAND)}; + static_cast(kvbc::keyTypes::CLIENT_COMMAND_TYPES::CLIENT_SCALING_COMMAND_STATUS)}, + // Main key update + std::string{kvbc::keyTypes::reconfiguration_rep_main_key}}; + + // Query latest state changes for replicas other than the sender for (uint16_t i = 0; i < first_client_id; i++) { if (i == sender_id) continue; - // 1. Handle TLS key exchange update - auto ke_csrep = buildReplicaStateReply(std::string{kvbc::keyTypes::reconfiguration_tls_exchange_key}, i); - if (ke_csrep.block_id > 0) rep.states.push_back(ke_csrep); - // 2. Handle scaling command - auto scale_csrep = buildReplicaStateReply(scaling_key_prefix, i); - if (scale_csrep.block_id > 0) rep.states.push_back(scale_csrep); - // 3. Handler scaling status update - auto scale_status_csrep = buildReplicaStateReply( - std::string{kvbc::keyTypes::reconfiguration_client_data_prefix, - static_cast(kvbc::keyTypes::CLIENT_COMMAND_TYPES::CLIENT_SCALING_COMMAND_STATUS)}, - i); - if (scale_status_csrep.block_id > 0) rep.states.push_back(scale_csrep); + for (auto& update_type : non_external_client_update_types) { + auto state_reply = buildReplicaStateReply(update_type, i); + if (state_reply.block_id > 0) { + LOG_INFO(GL, "Got State reply for replica id" << i); + rep.states.push_back(state_reply); + } + } } } concord::messages::serialize(rres.additional_data, rep); @@ -1034,7 +1043,7 @@ bool ReconfigurationHandler::handle(const messages::UnwedgeCommand& cmd, } bool ReconfigurationHandler::handle(const messages::UnwedgeStatusRequest& req, - uint64_t, + uint64_t bft_seq_num, uint32_t, const std::optional& ts, concord::messages::ReconfigurationResponse& rres) { @@ -1050,13 +1059,13 @@ bool ReconfigurationHandler::handle(const messages::UnwedgeStatusRequest& req, } } auto curr_epoch = bftEngine::EpochManager::instance().getSelfEpochNumber(); + // TODO: serialize this data using a struct instead of string concatenation std::string sig_data = std::to_string(ReplicaConfig::instance().getreplicaId()) + std::to_string(curr_epoch); auto sig_manager = bftEngine::impl::SigManager::instance(); - std::string sig(sig_manager->getMySigLength(), '\0'); - sig_manager->sign(sig_data.c_str(), sig_data.size(), sig.data()); + response.signature.resize(sig_manager->getMySigLength()); + sig_manager->sign(bft_seq_num, sig_data.c_str(), sig_data.size(), reinterpret_cast(response.signature.data())); response.can_unwedge = true; response.curr_epoch = curr_epoch; - response.signature = std::vector(sig.begin(), sig.end()); LOG_INFO(getLogger(), "Replica is ready to unwedge " << KVLOG(curr_epoch)); rres.response = response; return true; diff --git a/kvbc/src/v4blockchain/v4_blockchain.cpp b/kvbc/src/v4blockchain/v4_blockchain.cpp index b8ec36f9c8..0fb9012b17 100644 --- a/kvbc/src/v4blockchain/v4_blockchain.cpp +++ b/kvbc/src/v4blockchain/v4_blockchain.cpp @@ -99,6 +99,8 @@ BlockId KeyValueBlockchain::add(categorization::Updates &&updates) { if (block_id % 100 == 0) { v4_metrics_comp_.UpdateAggregator(); } + LOG_INFO(GL, "Adding block to DB with" << KVLOG(sequence_number, block_id)); + printCallStack(); return block_id; } diff --git a/kvbc/tools/object_store_utility/integrity_checker.cpp b/kvbc/tools/object_store_utility/integrity_checker.cpp index 73f1453d18..78d7f55bdf 100644 --- a/kvbc/tools/object_store_utility/integrity_checker.cpp +++ b/kvbc/tools/object_store_utility/integrity_checker.cpp @@ -16,7 +16,7 @@ #include "s3/config_parser.hpp" #include "bftengine/ReplicaConfig.hpp" #include "bftengine/ReplicasInfo.hpp" -#include "bftengine/SigManager.hpp" +#include "bftengine/ValidationOnlyIdentityManager.hpp" #include "bftengine/CheckpointInfo.hpp" #include "bcstatetransfer/BCStateTran.hpp" #include "direct_kv_storage_factory.h" @@ -52,13 +52,7 @@ void IntegrityChecker::initKeysConfig(const fs::path& keys_file) { repsInfo_ = new ReplicasInfo(config, true, false); - bftEngine::impl::SigManager::init(config.replicaId, - "", /*private key*/ - config.publicKeysOfReplicas, - KeyFormat::HexaDecimalStrippedFormat, - nullptr /*publicKeysOfClients*/, - KeyFormat::PemFormat, - *repsInfo_); + bftEngine::impl::ValidationOnlyIdentityManager::init(config.replicaId, config.publicKeysOfReplicas, *repsInfo_); } void IntegrityChecker::initS3Config(const fs::path& s3_file) { diff --git a/libs/communication/StateControl.hpp b/libs/communication/StateControl.hpp index 96d1af00fc..2595fe9c82 100644 --- a/libs/communication/StateControl.hpp +++ b/libs/communication/StateControl.hpp @@ -30,6 +30,9 @@ struct EnumHash { }; class StateControl { + private: + static constexpr const size_t replicaIdentityHistoryCount = 2; + public: using CallbackRegistry = concord::util::CallbackRegistry; enum class EventType { TLS_COMM, THIN_REPLICA_SERVER }; @@ -70,11 +73,11 @@ class StateControl { } return str; } - void setGetPeerPubKeyMethod(std::function m) { get_peer_pub_key_ = std::move(m); } + void setGetPeerPubKeyMethod(std::function(uint32_t)> m) { get_peer_pub_key_ = std::move(m); } - std::string getPeerPubKey(uint32_t id) { + std::array getPeerPubKey(uint32_t id) { if (get_peer_pub_key_) return get_peer_pub_key_(id); - return std::string(); + return {}; } private: @@ -100,6 +103,6 @@ class StateControl { std::mutex lock_comm_; // keeping function template to be void(uint32_t) for uniformity std::unordered_map, EnumHash> event_registry_; - std::function get_peer_pub_key_; + std::function(uint32_t)> get_peer_pub_key_; }; } // namespace bft::communication diff --git a/libs/communication/src/AsyncTlsConnection.cpp b/libs/communication/src/AsyncTlsConnection.cpp index 340346916b..b1e5bd09a9 100644 --- a/libs/communication/src/AsyncTlsConnection.cpp +++ b/libs/communication/src/AsyncTlsConnection.cpp @@ -415,7 +415,7 @@ bool AsyncTlsConnection::verifyCertificateClient(boost::asio::ssl::verify_contex return false; } auto [valid, _] = checkCertificate(*cert, expected_dest_id); - (void)_; // unused variable hack + UNUSED(_); // unused variable hack return valid; } @@ -447,16 +447,25 @@ std::pair AsyncTlsConnection::checkCertificate(X509& received_cer LOG_INFO(logger_, "Unable to validate certificate against the local storage, falling back to validate against the replica " "main key"); - std::string pem_pub_key = StateControl::instance().getPeerPubKey(peerId); - if (pem_pub_key.empty()) return std::make_pair(false, peerId); - if (ReplicaConfig::instance().replicaMsgSigningAlgo == SignatureAlgorithm::EdDSA) { + auto hex_pub_keys = StateControl::instance().getPeerPubKey(peerId); + for (auto& hex_pub_key : hex_pub_keys) { + std::string pem_pub_key = hex_pub_key; + if (pem_pub_key.empty()) { + continue; + } + ConcordAssertEQ(ReplicaConfig::instance().replicaMsgSigningAlgo, SignatureAlgorithm::EdDSA); if (getFormat(pem_pub_key) != concord::crypto::KeyFormat::PemFormat) { - pem_pub_key = EdDSAHexToPem(std::make_pair("", StateControl::instance().getPeerPubKey(peerId))).second; + pem_pub_key = EdDSAHexToPem(std::make_pair("", hex_pub_key)).second; + } + + // (2) Try to validate the certificate against the peer's public key + LOG_INFO(logger_, "Validating certificate was signed with public key: " << hex_pub_key); + res = verifyCertificate(received_cert, pem_pub_key); + if (res) { + break; } } - // (2) Try to validate the certificate against the peer's public key - res = verifyCertificate(received_cert, pem_pub_key); if (!res) return std::make_pair(false, peerId); // (3) If valid, exchange the stored certificate diff --git a/libs/crypto/digest_holder.hpp b/libs/crypto/digest_holder.hpp index 3ac141a8f3..dd7ec933ee 100644 --- a/libs/crypto/digest_holder.hpp +++ b/libs/crypto/digest_holder.hpp @@ -26,6 +26,7 @@ using BlockDigest = std::array; template >> class DigestHolder { public: + static size_t constexpr size() { return DIGEST_SIZE; } DigestHolder() { std::memset(d, 0, DIGEST_SIZE); } DigestHolder(unsigned char initVal) { std::memset(d, initVal, DIGEST_SIZE); } DigestHolder(const char* other) { std::memcpy(d, other, DIGEST_SIZE); } diff --git a/libs/crypto/openssl/EdDSASigner.hpp b/libs/crypto/openssl/EdDSASigner.hpp index f3f5fad68c..97c9e84eaf 100644 --- a/libs/crypto/openssl/EdDSASigner.hpp +++ b/libs/crypto/openssl/EdDSASigner.hpp @@ -35,7 +35,8 @@ class EdDSASigner : public ISigner { */ explicit EdDSASigner(const PrivateKeyType &privateKey) : privateKey_(privateKey) {} - size_t signBuffer(const concord::Byte *msg, size_t len, concord::Byte *signature) override { + size_t signBuffer(const concord::Byte *msg, size_t len, concord::Byte *signature) const override { + LOG_INFO(GL, "Signing with key " << getPrivKey()); UniquePKEY pkey(EVP_PKEY_new_raw_private_key( NID_ED25519, nullptr, privateKey_.getBytes().data(), privateKey_.getBytes().size())); ConcordAssertNE(pkey, nullptr); diff --git a/libs/crypto/openssl/EdDSAVerifier.hpp b/libs/crypto/openssl/EdDSAVerifier.hpp index c579c26945..d5934b3d28 100644 --- a/libs/crypto/openssl/EdDSAVerifier.hpp +++ b/libs/crypto/openssl/EdDSAVerifier.hpp @@ -38,7 +38,9 @@ class EdDSAVerifier : public IVerifier { EVP_PKEY_new_raw_public_key(NID_ED25519, nullptr, publicKey_.getBytes().data(), publicKey_.getBytes().size())}; UniqueContext ctx{EVP_MD_CTX_new()}; ConcordAssertEQ(EVP_DigestVerifyInit(ctx.get(), nullptr, nullptr, nullptr, pkey.get()), OPENSSL_SUCCESS); - return (OPENSSL_SUCCESS == EVP_DigestVerify(ctx.get(), sig, sigLen, msg, msgLen)); + auto result = OPENSSL_SUCCESS == EVP_DigestVerify(ctx.get(), sig, sigLen, msg, msgLen); + LOG_INFO(GL, "Verification complete" << KVLOG(getPubKey(), result)); + return result; } uint32_t signatureLength() const override { return Ed25519SignatureByteSize; } diff --git a/libs/crypto/signer.hpp b/libs/crypto/signer.hpp index e31fd0f0ad..407398561c 100644 --- a/libs/crypto/signer.hpp +++ b/libs/crypto/signer.hpp @@ -24,7 +24,7 @@ class ISigner { public: // This function's name need to be different from ISigner::sign, otherwise inheriting // classes will hide ISigner::sign - virtual size_t signBuffer(const Byte* dataIn, size_t dataLen, Byte* sigOutBuffer) = 0; + virtual size_t signBuffer(const Byte* dataIn, size_t dataLen, Byte* sigOutBuffer) const = 0; template size_t sign(const Container& dataIn, Byte* sigOutBuffer) { static_assert(sizeof(typename Container::value_type) == sizeof(Byte), diff --git a/libs/crypto/src/threshsign/ThresholdSignaturesTypes.cpp b/libs/crypto/src/threshsign/ThresholdSignaturesTypes.cpp index 6f2ac8b07f..4374934b6f 100644 --- a/libs/crypto/src/threshsign/ThresholdSignaturesTypes.cpp +++ b/libs/crypto/src/threshsign/ThresholdSignaturesTypes.cpp @@ -66,20 +66,18 @@ void Cryptosystem::generateNewPseudorandomKeys() { auto [signers, verifier] = factory->newRandomSigners(threshold_, numSigners_); verificationKeys_.clear(); - verificationKeys_.resize(static_cast(numSigners_ + 1)); - verificationKeys_[0] = ""; // Account for 1-indexing of signer IDs. - for (uint16_t i = 1; i <= numSigners_; ++i) { + verificationKeys_.resize(static_cast(numSigners_)); + for (uint16_t i = 0; i < numSigners_; ++i) { verificationKeys_[i] = verifier->getShareVerificationKey(static_cast(i)).toString(); } privateKeys_.clear(); - privateKeys_.resize(static_cast(numSigners_ + 1)); - privateKeys_[0] = ""; // Account for 1-indexing of signer IDs. - for (uint16_t i = 1; i <= numSigners_; ++i) { + privateKeys_.resize(static_cast(numSigners_)); + for (uint16_t i = 0; i < numSigners_; ++i) { privateKeys_[i] = signers[i]->getShareSecretKey().toString(); } - signerID_ = NID; + signerID_ = INVALID_SIGNER_ID; } std::pair Cryptosystem::generateNewKeyPair() { @@ -99,7 +97,7 @@ std::string Cryptosystem::getSystemPublicKey() const { std::vector Cryptosystem::getSystemVerificationKeys() const { std::vector output; - if (verificationKeys_.size() != static_cast(numSigners_ + 1)) { + if (verificationKeys_.size() != static_cast(numSigners_)) { throw std::runtime_error( "Verification keys have not been" " generated or loaded for this cryptosystem."); @@ -111,7 +109,7 @@ std::vector Cryptosystem::getSystemVerificationKeys() const { std::vector Cryptosystem::getSystemPrivateKeys() const { std::vector output; - if (privateKeys_.size() != static_cast(numSigners_ + 1)) { + if (privateKeys_.size() != static_cast(numSigners_)) { throw std::runtime_error( "Private keys have not been" " generated or loaded for this cryptosystem."); @@ -122,41 +120,39 @@ std::vector Cryptosystem::getSystemPrivateKeys() const { } std::string Cryptosystem::getPrivateKey(uint16_t signerIndex) const { - if ((signerIndex < 1) || (signerIndex > numSigners_)) - throw std::out_of_range(__PRETTY_FUNCTION__ + std::string("Signer index out of range: ") + + if ((signerIndex < 0) || (signerIndex >= numSigners_)) + throw std::out_of_range(__PRETTY_FUNCTION__ + std::string(" Signer index out of range: ") + std::to_string(signerIndex)); - if (privateKeys_.size() == static_cast(numSigners_ + 1)) { - return privateKeys_[signerIndex]; - } else if ((privateKeys_.size() == 1) && (signerID_ == signerIndex)) { - return privateKeys_.front(); + if (privateKeys_.size() < 1) { + throw std::runtime_error( + "Private keys have not been" + " generated or loaded for this cryptosystem."); } - throw std::runtime_error( - "Private keys have not been" - " generated or loaded for this cryptosystem."); + return privateKeys_[signerIndex]; } void Cryptosystem::loadKeys(const std::string& publicKey, const std::vector& verificationKeys) { validatePublicKey(publicKey); - if (verificationKeys.size() != static_cast(numSigners_ + 1)) { + if (verificationKeys.size() != static_cast(numSigners_)) { throw std::runtime_error( "Incorrect number of verification keys provided: " + std::to_string(verificationKeys.size()) + " (expected " + - std::to_string(numSigners_ + 1) + ")."); + std::to_string(numSigners_) + ")."); } - for (size_t i = 1; i <= numSigners_; ++i) validateVerificationKey(verificationKeys[i]); + for (size_t i = 0; i < numSigners_; ++i) validateVerificationKey(verificationKeys[i]); verificationKeys_.clear(); privateKeys_.clear(); publicKey_ = publicKey; - signerID_ = NID; + signerID_ = INVALID_SIGNER_ID; verificationKeys_ = verificationKeys; } void Cryptosystem::loadPrivateKey(uint16_t signerIndex, const std::string& key) { - if ((signerIndex < 1) || (signerIndex > numSigners_)) + if (signerIndex > numSigners_) throw std::out_of_range(__PRETTY_FUNCTION__ + std::string("Signer index out of range: ") + std::to_string(signerIndex)); @@ -184,7 +180,7 @@ IThresholdVerifier* Cryptosystem::createThresholdVerifier(uint16_t threshold) { "Attempting to create a threshold" " verifier for a cryptosystem with no public key loaded."); } - if (verificationKeys_.size() != static_cast(numSigners_ + 1)) { + if (verificationKeys_.size() != static_cast(numSigners_)) { throw std::runtime_error( "Attempting to create a threshold" " verifier for a cryptosystem without verification keys loaded."); @@ -210,8 +206,6 @@ IThresholdSigner* Cryptosystem::createThresholdSigner() { } std::unique_ptr factory(createThresholdFactory()); - // Note we add 1 to the signer ID because IThresholdSigner seems to use a - // convention in which signer IDs are 1-indexed. return factory->newSigner(signerID_, privateKeys_.front().c_str()); } @@ -292,10 +286,10 @@ void Cryptosystem::writeConfiguration(std::ostream& output, const std::string& p output << prefix << "_cryptosystem_public_key: " << getSystemPublicKey() << "\n"; std::vector verificationKeys = getSystemVerificationKeys(); output << prefix << "_cryptosystem_verification_keys:\n"; - for (uint16_t i = 1; i <= numReplicas; ++i) output << " - " << verificationKeys[i] << "\n"; + for (uint16_t i = 0; i < numReplicas; ++i) output << " - " << verificationKeys[i] << "\n"; output << "\n"; - output << prefix << "_cryptosystem_private_key: " << getPrivateKey((uint16_t)(replicaId + 1)) << "\n\n"; + output << prefix << "_cryptosystem_private_key: " << getPrivateKey((uint16_t)(replicaId)) << "\n\n"; } Cryptosystem* Cryptosystem::fromConfiguration(std::istream& input, @@ -314,16 +308,21 @@ Cryptosystem* Cryptosystem::fromConfiguration(std::istream& input, if (type == MULTISIG_EDDSA_SCHEME) threshold = yaml::readValue(input, prefix + "_cryptosystem_threshold"); thrPublicKey = yaml::readValue(input, prefix + "_cryptosystem_public_key"); - thrVerificationKeys = yaml::readCollection(input, prefix + "_cryptosystem_verification_keys"); + auto parsedHexKeys = yaml::readCollection(input, prefix + "_cryptosystem_verification_keys"); + if (thrVerificationKeys.empty()) { + thrVerificationKeys = parsedHexKeys; + } if (thrVerificationKeys.size() != numSigners) throw std::runtime_error("expected " + std::to_string(numSigners) + std::string(" verification keys, got: ") + std::to_string(thrVerificationKeys.size())); - thrPrivateKey = yaml::readValue(input, prefix + "_cryptosystem_private_key"); + auto parsedHexPrivateKey = yaml::readValue(input, prefix + "_cryptosystem_private_key"); + if (thrPrivateKey.empty()) { + thrPrivateKey = parsedHexPrivateKey; + } Cryptosystem* sys = new Cryptosystem(type, subtype, numSigners, threshold); - thrVerificationKeys.insert(thrVerificationKeys.begin(), ""); sys->loadKeys(thrPublicKey, thrVerificationKeys); sys->loadPrivateKey(signerIndex, thrPrivateKey); diff --git a/libs/crypto/src/threshsign/eddsa/EdDSAMultisigSigner.cpp b/libs/crypto/src/threshsign/eddsa/EdDSAMultisigSigner.cpp index 89e4e5a702..43dbf4f10c 100644 --- a/libs/crypto/src/threshsign/eddsa/EdDSAMultisigSigner.cpp +++ b/libs/crypto/src/threshsign/eddsa/EdDSAMultisigSigner.cpp @@ -16,7 +16,10 @@ using concord::crypto::openssl::EdDSASigner; EdDSAMultisigSigner::EdDSAMultisigSigner(const EdDSAThreshsignPrivateKey &privateKey, const uint32_t id) - : EdDSASigner{privateKey}, publicKey_{}, id_{id} {} + : EdDSASigner{privateKey}, publicKey_{}, id_{id} { + LOG_INFO(EDDSA_MULTISIG_LOG, "created eddsa signer with " << KVLOG(id_, getPrivKey())); + printCallStack(); +} int EdDSAMultisigSigner::requiredLengthForSignedData() const { return sizeof(SingleEdDSASignature); } diff --git a/libs/crypto/src/threshsign/eddsa/EdDSAMultisigVerifier.cpp b/libs/crypto/src/threshsign/eddsa/EdDSAMultisigVerifier.cpp index 84291df3ed..01979997d4 100644 --- a/libs/crypto/src/threshsign/eddsa/EdDSAMultisigVerifier.cpp +++ b/libs/crypto/src/threshsign/eddsa/EdDSAMultisigVerifier.cpp @@ -17,7 +17,7 @@ int EdDSASignatureAccumulator::add(const char *sigShareWithId, int len) { ConcordAssertEQ(len, static_cast(sizeof(SingleEdDSASignature))); auto &singleSignature = *reinterpret_cast(sigShareWithId); - if (singleSignature.id == 0 || singleSignature.id > verifier_.maxShareID()) { + if (singleSignature.id > verifier_.maxShareID()) { LOG_ERROR(EDDSA_MULTISIG_LOG, "Invalid signer id" << KVLOG(singleSignature.id, verifier_.maxShareID())); return static_cast(signatures_.size()); } @@ -27,13 +27,13 @@ int EdDSASignatureAccumulator::add(const char *sigShareWithId, int len) { reinterpret_cast(expectedMsgDigest_.data()), expectedMsgDigest_.size(), singleSignature); if (!result) { invalidShares_.insert(static_cast(singleSignature.id)); + LOG_INFO(EDDSA_MULTISIG_LOG, "Share id: " << singleSignature.id << " is invalid"); } - LOG_DEBUG(EDDSA_MULTISIG_LOG, "Share id: " << singleSignature.id << "Invalid"); } auto result = signatures_.insert({singleSignature.id, singleSignature}); if (result.second) { - LOG_DEBUG(EDDSA_MULTISIG_LOG, "Added " << KVLOG(this, singleSignature.id, signatures_.size())); + LOG_INFO(EDDSA_MULTISIG_LOG, "Added " << KVLOG(this, singleSignature.id, signatures_.size())); } return static_cast(signatures_.size()); } @@ -43,7 +43,7 @@ void EdDSASignatureAccumulator::setExpectedDigest(const unsigned char *msg, int } size_t EdDSASignatureAccumulator::getFullSignedData(char *outThreshSig, int threshSigLen) { - LOG_DEBUG(EDDSA_MULTISIG_LOG, KVLOG(threshSigLen, signatures_.size())); + LOG_INFO(EDDSA_MULTISIG_LOG, KVLOG(threshSigLen, signatures_.size())); ConcordAssertGE(static_cast(threshSigLen), static_cast(signatures_.size()) * sizeof(SingleEdDSASignature)); size_t offset = 0; @@ -70,7 +70,7 @@ EdDSAMultisigVerifier::EdDSAMultisigVerifier(const std::vector & const size_t signersCount, const size_t threshold) : verifiers_(verifiers), signersCount_(signersCount), threshold_(threshold) { - ConcordAssertEQ(verifiers.size(), signersCount + 1); + ConcordAssertEQ(verifiers.size(), signersCount); LOG_DEBUG(EDDSA_MULTISIG_LOG, KVLOG(this, verifiers_.size(), threshold_)); } @@ -81,7 +81,7 @@ IThresholdAccumulator *EdDSAMultisigVerifier::newAccumulator(bool withShareVerif bool EdDSAMultisigVerifier::verifySingleSignature(const concord::Byte *msg, size_t msgLen, const SingleEdDSASignature &signature) const { - if (signature.id == 0) { + if (signature.id >= verifiers_.size()) { return false; } return verifiers_[signature.id].verifyBuffer( @@ -104,18 +104,18 @@ bool EdDSAMultisigVerifier::verify(const char *msg, int msgLen, const char *sig, for (int i = 0; i < static_cast(signatureCountInBuffer); i++) { auto ¤tSignature = allSignatures[i]; - if (currentSignature.id == 0 || currentSignature.id >= verifiers_.size()) { + if (currentSignature.id >= verifiers_.size()) { LOG_ERROR(EDDSA_MULTISIG_LOG, "Invalid signer id" << KVLOG(currentSignature.id, verifiers_.size())); continue; } auto result = verifySingleSignature(reinterpret_cast(msg), msgLenUnsigned, currentSignature); - LOG_DEBUG(EDDSA_MULTISIG_LOG, "Verified id: " << KVLOG(currentSignature.id, result)); + LOG_INFO(EDDSA_MULTISIG_LOG, "Verified id: " << KVLOG(currentSignature.id, result)); validSignatureCount += result == true; } bool result = validSignatureCount >= threshold_; - LOG_DEBUG(EDDSA_MULTISIG_LOG, KVLOG(validSignatureCount, threshold_)); + LOG_INFO(EDDSA_MULTISIG_LOG, KVLOG(validSignatureCount, threshold_)); return result; } @@ -126,4 +126,7 @@ const IPublicKey &EdDSAMultisigVerifier::getPublicKey() const { return verifiers const IShareVerificationKey &EdDSAMultisigVerifier::getShareVerificationKey(ShareID signer) const { return verifiers_[static_cast(signer)].publicKey_; } -size_t EdDSAMultisigVerifier::maxShareID() const { return signersCount_ + 1; } +size_t EdDSAMultisigVerifier::maxShareID() const { return signersCount_; } +const concord::crypto::IVerifier &EdDSAMultisigVerifier::getVerifier(size_t verifier) const { + return verifiers_[verifier]; +} diff --git a/libs/crypto/src/threshsign/eddsa/test/TestEdDSAMultisig.cpp b/libs/crypto/src/threshsign/eddsa/test/TestEdDSAMultisig.cpp index 5e204a582d..46e221ed5f 100644 --- a/libs/crypto/src/threshsign/eddsa/test/TestEdDSAMultisig.cpp +++ b/libs/crypto/src/threshsign/eddsa/test/TestEdDSAMultisig.cpp @@ -62,8 +62,8 @@ TEST_F(EdDSAMultisigTest, TestSingleVerificationUsingFactory) { auto eddsaPrivateKey = dynamic_cast(privateKey.get()); auto eddsaPublicKey = dynamic_cast(publicKey.get()); - auto* signer = factory_.newSigner(1, eddsaPrivateKey->toString().c_str()); - auto* verifier = factory_.newVerifier(1, 1, "", {"", eddsaPublicKey->toString()}); + auto* signer = factory_.newSigner(0, eddsaPrivateKey->toString().c_str()); + auto* verifier = factory_.newVerifier(1, 1, "", {eddsaPublicKey->toString()}); auto accumulator = verifier->newAccumulator(false); auto msg = testMsgDigest(); @@ -88,9 +88,9 @@ TEST_F(EdDSAMultisigTest, TestSignatureAccumulation) { const std::string digest = testMsgDigest(); accumulator->setExpectedDigest(reinterpret_cast(digest.data()), (int)digest.size()); - std::vector signatures(signers.size() - 1); + std::vector signatures(signers.size()); for (size_t i = 0; i < signatures.size(); i++) { - signers[i + 1]->signData( + signers[i]->signData( digest.data(), (int)digest.size(), reinterpret_cast(&signatures[i]), sizeof(SingleEdDSASignature)); } @@ -122,12 +122,12 @@ TEST_F(EdDSAMultisigTest, TestValidMultiSignatureSmallThreshold) { auto accumulator = verifier->newAccumulator(false); accumulator->setExpectedDigest(reinterpret_cast(digest.data()), (int)digest.size()); - std::vector signatures(signers.size() - 1); + std::vector signatures(signers.size()); for (size_t i = 0; i < signatures.size(); i++) { - signers[i + 1]->signData(digest.data(), - static_cast(digest.size()), - reinterpret_cast(&signatures[i]), - sizeof(SingleEdDSASignature)); + signers[i]->signData(digest.data(), + static_cast(digest.size()), + reinterpret_cast(&signatures[i]), + sizeof(SingleEdDSASignature)); } // Make sure that verification does not depend on order, as signatures do not arrive in order std::random_shuffle(signatures.begin(), signatures.end()); diff --git a/libs/crypto/threshsign/ThresholdSignaturesTypes.h b/libs/crypto/threshsign/ThresholdSignaturesTypes.h index 0c7646f943..2bcfb3f7b7 100644 --- a/libs/crypto/threshsign/ThresholdSignaturesTypes.h +++ b/libs/crypto/threshsign/ThresholdSignaturesTypes.h @@ -44,14 +44,13 @@ class Cryptosystem { uint16_t numSigners_{0}; uint16_t threshold_{0}; + static constexpr const uint16_t INVALID_SIGNER_ID = UINT16_MAX; + // If only one signer's private key is known and stored in this cryptosystem, // this field records that signer's ID; otherwise (if no or all private keys // are known to this cryptosystem), this field stores Cryptosystem::NID to // represent it is inapplicable. - uint16_t signerID_{0}; - - // Note that 0 is not a valid signer ID because signer IDs are 1-indexed. - static const uint16_t NID = 0; + uint16_t signerID_{INVALID_SIGNER_ID}; std::string publicKey_{""}; std::vector verificationKeys_; diff --git a/libs/reconfiguration/cmf/concord.cmf b/libs/reconfiguration/cmf/concord.cmf index a5c2546adf..9a053d6cce 100644 --- a/libs/reconfiguration/cmf/concord.cmf +++ b/libs/reconfiguration/cmf/concord.cmf @@ -230,6 +230,7 @@ Msg ReplicaMainKeyUpdate 64 { string key string format uint8 algorithm + uint64 seq_num } Msg ClientStateReply 39 { diff --git a/libs/reconfiguration/src/dispatcher.cpp b/libs/reconfiguration/src/dispatcher.cpp index 102b86f39a..54d148d052 100644 --- a/libs/reconfiguration/src/dispatcher.cpp +++ b/libs/reconfiguration/src/dispatcher.cpp @@ -12,6 +12,7 @@ #include "reconfiguration/dispatcher.hpp" #include "util/kvstream.h" +#include using namespace concord::messages; @@ -40,37 +41,10 @@ ReconfigurationResponse Dispatcher::dispatch(const ReconfigurationRequest& reque rresp.success = true; auto sender_id = request.sender; executions_++; + auto partial_ordered_handlers = boost::join(pre_reconfig_handlers_, reconfig_handlers_); + auto all_ordered_handlers = boost::join(partial_ordered_handlers, post_reconfig_handlers_); try { - // Run pre-reconfiguration handlers - for (auto& handler : pre_reconfig_handlers_) { - // Each reconfiguration handler handles only what it can validate - if (!handler->verifySignature(sender_id, ser_data, ser_sig)) { - error_msg.error_msg = "Invalid signature"; - continue; - } - error_msg.error_msg.clear(); - valid = true; - rresp.success &= - std::visit([&](auto&& arg) { return handleRequest(arg, sequence_num, sender_id, timestamp, rresp, handler); }, - request.command); - } - - // Run regular reconfiguration handlers - for (auto& handler : reconfig_handlers_) { - // Each reconfiguration handler handles only what it can validate - if (!handler->verifySignature(sender_id, ser_data, ser_sig)) { - error_msg.error_msg = "Invalid signature"; - continue; - } - error_msg.error_msg.clear(); - valid = true; - rresp.success &= - std::visit([&](auto&& arg) { return handleRequest(arg, sequence_num, sender_id, timestamp, rresp, handler); }, - request.command); - } - - // Run post-reconfiguration handlers - for (auto& handler : post_reconfig_handlers_) { + for (auto& handler : all_ordered_handlers) { // Each reconfiguration handler handles only what it can validate if (!handler->verifySignature(sender_id, ser_data, ser_sig)) { error_msg.error_msg = "Invalid signature"; @@ -107,6 +81,7 @@ Dispatcher::Dispatcher() failures_{component_.RegisterCounter("failures")} { component_.Register(); component_.UpdateAggregator(); + LOG_INFO(getLogger(), "Dispatcher initialized"); } } // namespace concord::reconfiguration diff --git a/libs/util/SerializableByteArray.hpp b/libs/util/SerializableByteArray.hpp index 6e845f4e4f..941e9aff0a 100644 --- a/libs/util/SerializableByteArray.hpp +++ b/libs/util/SerializableByteArray.hpp @@ -15,12 +15,13 @@ #include #include "util/assertUtils.hpp" #include "util/serializable.hpp" +#include "util/types.hpp" template class SerializableByteArray : concord::serialize::IStringSerializable { public: static constexpr const size_t ByteSize = ByteCount; - using ByteArray = std::array; + using ByteArray = std::array; SerializableByteArray(const ByteArray& bytes) : bytes_(bytes) {} virtual ~SerializableByteArray() = default; diff --git a/libs/util/assertUtils.hpp b/libs/util/assertUtils.hpp index 8101f83c7d..43e4682165 100644 --- a/libs/util/assertUtils.hpp +++ b/libs/util/assertUtils.hpp @@ -23,6 +23,51 @@ #include "util/kvstream.h" inline void printCallStack() { + /*const uint32_t MAX_FRAMES = 100; + void *addrlist[MAX_FRAMES]; + int addrLen = backtrace(addrlist, MAX_FRAMES); + if (addrLen) { + char **symbolsList = backtrace_symbols(addrlist, addrLen); + if (symbolsList) { + std::ostringstream os; + const size_t MAX_FUNC_NAME_SIZE = 256; + // Iterate over the returned symbol lines. Skip the first, it is the address of this function. + for (int i = 1; i < addrLen; i++) { + char *beginName = nullptr, *beginOffset = nullptr, *endOffset = nullptr; + for (char *ptr = symbolsList[i]; *ptr; ++ptr) { + if (*ptr == '(') + beginName = ptr; + else if (*ptr == '+') + beginOffset = ptr; + else if (*ptr == ')' && beginOffset) { + endOffset = ptr; + break; + } + } + if (beginName && beginOffset && endOffset && beginName < beginOffset) { + *beginName++ = '\0'; + *beginOffset++ = '\0'; + *endOffset = '\0'; + int status; + size_t demangledSize; + char *ret = abi::__cxa_demangle(beginName, nullptr, &demangledSize, &status); + if (status == 0) { + if (demangledSize > MAX_FUNC_NAME_SIZE) { + ret[MAX_FUNC_NAME_SIZE] = '\0'; + } + os << " [bt] " << ret << "+" << beginOffset << std::endl; + } + free(ret); + } + } + LOG_INFO(GL, "\n" << os.str()); + std::free(symbolsList); + } + }*/ + return; +} + +inline void printCallStack2() { const uint32_t MAX_FRAMES = 100; void *addrlist[MAX_FRAMES]; int addrLen = backtrace(addrlist, MAX_FRAMES); @@ -60,10 +105,11 @@ inline void printCallStack() { free(ret); } } - LOG_FATAL(GL, "\n" << os.str()); + LOG_INFO(GL, "\n" << os.str()); std::free(symbolsList); } } + return; } #define PRINT_DATA_AND_ASSERT_BOOL_EXPR(expr1, expr2, assertMacro) \ diff --git a/tests/apollo/test_skvbc_commit_path.py b/tests/apollo/test_skvbc_commit_path.py index cd8a74cb13..233dd969bd 100644 --- a/tests/apollo/test_skvbc_commit_path.py +++ b/tests/apollo/test_skvbc_commit_path.py @@ -271,4 +271,4 @@ async def test_fast_path_after_view_change(self, bft_network, tracker): await bft_network.wait_for_consensus_path( path_type=ConsensusPathType.OPTIMISTIC_FAST, run_ops=lambda: skvbc.send_n_kvs_sequentially(int(1.1 * EVALUATION_PERIOD_SEQUENCES)), - threshold=num_ops) + threshold=num_ops) \ No newline at end of file diff --git a/tests/apollo/test_skvbc_dbsnapshot.py b/tests/apollo/test_skvbc_dbsnapshot.py index d8151d60a7..ad3d591def 100644 --- a/tests/apollo/test_skvbc_dbsnapshot.py +++ b/tests/apollo/test_skvbc_dbsnapshot.py @@ -595,6 +595,9 @@ async def test_db_checkpoint_creation_with_wedge(self, bft_network): op = operator.Operator( bft_network.config, client, bft_network.builddir) await op.wedge() + #from time import sleep + #sleep(1) + #return await bft_network.wait_for_stable_checkpoint( bft_network.all_replicas(), stable_seqnum = (checkpoint_before + 2) * 150) await self.validate_stop_on_wedge_point(bft_network, skvbc=skvbc, fullWedge=True) # verify that snapshot is created on wedge point diff --git a/tests/apollo/test_skvbc_reconfiguration.py b/tests/apollo/test_skvbc_reconfiguration.py index 815519d3c3..ba3d889ced 100644 --- a/tests/apollo/test_skvbc_reconfiguration.py +++ b/tests/apollo/test_skvbc_reconfiguration.py @@ -577,7 +577,7 @@ async def test_tls_exchange_replicas_replicas_with_ror(self, bft_network): for i in range(301): # Produce 301 new blocks await skvbc.send_write_kv_set() # Now, lets switch keys to all replicas - exchanged_replicas = bft_network.all_replicas() + exchanged_replicas = bft_network.all_replicas(without={1,2,3,4,5,6}) await self.run_replica_tls_key_exchange_cycle(bft_network, exchanged_replicas, affected_replicas=[r for r in range(bft_network.config.n + 1)]) bft_network.copy_certs_from_server_to_clients(7) bft_network.restart_clients(False, False) diff --git a/tests/apollo/test_skvbc_state_transfer.py b/tests/apollo/test_skvbc_state_transfer.py index 5358b1f950..3110c270b4 100644 --- a/tests/apollo/test_skvbc_state_transfer.py +++ b/tests/apollo/test_skvbc_state_transfer.py @@ -7,7 +7,7 @@ # # This product may include a number of subcomponents with separate copyright # notices and license terms. Your use of these subcomponents is subject to the -# terms and conditions of the subcomponent's license, as noted in the LICENSE +# terms and conditions of the subcomponent's license, as noted in the LICENStE # file. import os.path import random @@ -287,6 +287,7 @@ async def test_state_transfer_rvt_validity_after_pruning(self, bft_network): # Wait for the RVT root values to be in sync before the pruning await bft_network.wait_for_replicas_rvt_root_values_to_be_in_sync(bft_network.all_replicas()) + return # Get the minimal latest pruneable block among all replicas client = bft_network.random_client() op = operator.Operator(bft_network.config, client, bft_network.builddir) diff --git a/tests/apollo/util/bft.py b/tests/apollo/util/bft.py index bea7ee5a11..d1e6981c02 100644 --- a/tests/apollo/util/bft.py +++ b/tests/apollo/util/bft.py @@ -247,7 +247,7 @@ async def test_with_bft_network(): return decorator MAX_MSG_SIZE = 64*1024 # 64k -REQ_TIMEOUT_MILLI = 5000 +REQ_TIMEOUT_MILLI = 5000 * 2 RETRY_TIMEOUT_MILLI = 250 METRICS_TIMEOUT_SEC = 5 @@ -519,6 +519,7 @@ async def change_configuration(self, config, generate_tls=False, use_unified_cer # Generate certificates for replicas, clients, and reserved clients self.generate_tls_certs(self.num_total_replicas() + config.num_clients + RESERVED_CLIENTS_QUOTA + generate_cre, use_unified_certs=use_unified_certs) + @log_call def restart_clients(self, generate_tx_signing_keys=True, restart_replicas=True): with log.start_action(action_type="restart_clients", generate_tx_signing_keys=generate_tx_signing_keys, restart_replicas=restart_replicas): @@ -623,6 +624,7 @@ def _create_clients(self): def _create_new_client(self, client_class, client_id): config = self._bft_config(client_id) + log_message(message_type=f"Creating client {client_id}", config=str(config)) ro_replicas = [r.id for r in self.ro_replicas] return client_class(config, self.replicas, self.background_nursery, ro_replicas=ro_replicas) diff --git a/tests/apollo/util/pyclient/bft_client.py b/tests/apollo/util/pyclient/bft_client.py index 9a34e666ed..514efc1567 100644 --- a/tests/apollo/util/pyclient/bft_client.py +++ b/tests/apollo/util/pyclient/bft_client.py @@ -123,7 +123,6 @@ def All(cls, config, replicas): def __str__(self): return f"n: {self.replicas}, m: {self.required}" - class BftClient(ABC): COUNTER = WriteCounter(val=int(1e6)) @@ -299,7 +298,7 @@ async def write_batch(self, msg_batch, batch_seq_nums=None, m_of_n_quorum=None, self.client_id, msg_seq_num, False, self.config.req_timeout_milli, batch_cid, msg, 0, True, reconfiguration=False, span_context=b'', signature=signature, batch_index=n)]) - data = bft_msgs.pack_batch_request(self.client_id, batch_size, msg_data, batch_cid) + data = bft_msgs.pack_batch_request(self.client_id, batch_size, msg_data, batch_batch_cid) if m_of_n_quorum is None: m_of_n_quorum = MofNQuorum.LinearizableQuorum(self.config, [r.id for r in self.replicas]) @@ -311,7 +310,7 @@ async def write_batch(self, msg_batch, batch_seq_nums=None, m_of_n_quorum=None, return await self._send_receive_loop(data, False, m_of_n_quorum, batch_size * self.config.retry_timeout_milli / 1000, no_retries=no_retries) except trio.TooSlowError: - raise trio.TooSlowError(f"client_id {self.client_id}, for batch msg {batch_cid} {batch_seq_nums}") + raise trio.TooSlowError(f"client_id {self.client_id}, for batch msg {batch_batch_cid} {batch_seq_nums}") finally: pass diff --git a/tests/apollo/util/test_base.py b/tests/apollo/util/test_base.py index 24ce5f10a2..e7669cb3cf 100644 --- a/tests/apollo/util/test_base.py +++ b/tests/apollo/util/test_base.py @@ -87,7 +87,6 @@ async def wrapper(*args, **kwargs): return decorator - def repeat_test(max_repeats: int, break_on_first_failure: bool, break_on_first_success: bool, test_name=None): """ Runs a test max_repeats times when both break_on_first_failure and break_on_first_success et to False. diff --git a/tests/config/test_comm_config.cpp b/tests/config/test_comm_config.cpp index 556a3a934a..642c2afc1b 100644 --- a/tests/config/test_comm_config.cpp +++ b/tests/config/test_comm_config.cpp @@ -54,7 +54,7 @@ void TestCommConfig::GetReplicaConfig(uint16_t replica_id, auto sys = inputReplicaKeyfileMultisig(key_file_name, *out_config); if (sys) { std::unique_ptr up(sys); - bftEngine::CryptoManager::instance(std::move(up)); + bftEngine::CryptoManager::init(std::move(up)); } } diff --git a/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp b/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp index 580274e34e..88c4ec04dc 100644 --- a/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp +++ b/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp @@ -116,6 +116,7 @@ void InternalCommandsHandler::execute(InternalCommandsHandler::ExecutionRequests bool isBlockAccumulationEnabled = ((requests.size() > 1) && (req.flags & bftEngine::MsgFlag::HAS_PRE_PROCESSED_FLAG)); sequenceNum = req.executionSequenceNum; + LOG_INFO(GL, KVLOG(req.clientId, req.cid, req.requestSequenceNum)); res = executeWriteCommand(req.requestSize, req.request, req.executionSequenceNum, @@ -125,7 +126,10 @@ void InternalCommandsHandler::execute(InternalCommandsHandler::ExecutionRequests req.outActualReplySize, isBlockAccumulationEnabled, verUpdates, - merkleUpdates); + merkleUpdates, + std::stoi(req.cid), + req.requestSequenceNum, + req.clientId); } if (res != OperationResult::SUCCESS) LOG_WARN(m_logger, "Command execution failed!"); @@ -263,6 +267,23 @@ std::optional> InternalCommandsHandler::getBl } } + // Handle main key updates which should be ignored by the tests + for (auto &filteredCategory : {kConcordReconfigurationCategoryId}) { + const auto concordUpdates = updates->categoryUpdates(filteredCategory); + if (concordUpdates) { + const auto &u = std::get(concordUpdates->get()); + for (const auto &[key, valueWithFlags] : u.kv) { + if (key[0] == concord::kvbc::keyTypes::reconfiguration_rep_main_key) { + // Test categories and deployment categories are expected to be mutually exclusive + ConcordAssert(ret.empty()); + ret[s_ignoreBlockStr] = ""; + return ret; + } + } + } + } + + return ret; } @@ -404,10 +425,16 @@ OperationResult InternalCommandsHandler::executeWriteCommand(uint32_t requestSiz uint32_t &outReplySize, bool isBlockAccumulationEnabled, VersionedUpdates &blockAccumulatedVerUpdates, - BlockMerkleUpdates &blockAccumulatedMerkleUpdates) { + BlockMerkleUpdates &blockAccumulatedMerkleUpdates, + uint64_t batchCid, + uint64_t requestId, + uint16_t clientId) { static_assert(sizeof(*request) == sizeof(uint8_t), "Byte pointer type used by bftEngine::IRequestsHandler::ExecutionRequest is incompatible with byte " "pointer type used by CMF."); + const bool isFirstClientRequest = m_clientToMaxExecutedReqId.find(clientId) == m_clientToMaxExecutedReqId.end(); + const auto lastExecutedId = isFirstClientRequest ? 0 : m_clientToMaxExecutedReqId[clientId]; + const bool isBatchedRequest = batchCid != requestId; const uint8_t *request_buffer_as_uint8 = reinterpret_cast(request); if (!(flags & MsgFlag::HAS_PRE_PROCESSED_FLAG)) { auto res = verifyWriteCommand(requestSize, request_buffer_as_uint8, maxReplySize, outReplySize); @@ -426,7 +453,8 @@ OperationResult InternalCommandsHandler::executeWriteCommand(uint32_t requestSiz << " READ_ONLY_FLAG=" << ((flags & MsgFlag::READ_ONLY_FLAG) != 0 ? "true" : "false") << " PRE_PROCESS_FLAG=" << ((flags & MsgFlag::PRE_PROCESS_FLAG) != 0 ? "true" : "false") << " HAS_PRE_PROCESSED_FLAG=" << ((flags & MsgFlag::HAS_PRE_PROCESSED_FLAG) != 0 ? "true" : "false") - << " BLOCK_ACCUMULATION_ENABLED=" << isBlockAccumulationEnabled); + << " BLOCK_ACCUMULATION_ENABLED=" << isBlockAccumulationEnabled + << KVLOG(clientId, batchCid, requestId, isFirstClientRequest, lastExecutedId)); BlockId currBlock = m_storage->getLastBlockId(); // Look for conflicts @@ -447,6 +475,15 @@ OperationResult InternalCommandsHandler::executeWriteCommand(uint32_t requestSiz } } + if (isBatchedRequest) { + // The batched requests all share batchCid + // Batched requests are handled by the preprocessor and should not be sent more than once + hasConflict = hasConflict || (!isFirstClientRequest && m_clientToMaxExecutedReqId[clientId] > batchCid); + } + else { + hasConflict = hasConflict || (!isFirstClientRequest && m_clientToMaxExecutedReqId[clientId] >= requestId); + } + if (!hasConflict) { if (isBlockAccumulationEnabled) { // If Block Accumulation is enabled then blocks are added after all requests are processed @@ -463,11 +500,15 @@ OperationResult InternalCommandsHandler::executeWriteCommand(uint32_t requestSiz SKVBCReply reply; reply.reply = SKVBCWriteReply(); SKVBCWriteReply &write_rep = std::get(reply.reply); - write_rep.success = (!hasConflict); - if (!hasConflict) + write_rep.success = !hasConflict; + if (!hasConflict) { write_rep.latest_block = currBlock + 1; - else + auto [iter, isNew] = m_clientToMaxExecutedReqId.emplace(clientId, 0); + UNUSED(isNew); + m_clientToMaxExecutedReqId[clientId] = std::max(m_clientToMaxExecutedReqId[clientId], batchCid); + } else { write_rep.latest_block = currBlock; + } std::vector serialized_reply; serialize(serialized_reply, reply); @@ -478,8 +519,8 @@ OperationResult InternalCommandsHandler::executeWriteCommand(uint32_t requestSiz if (!isBlockAccumulationEnabled) LOG_INFO(m_logger, - "ConditionalWrite message handled; writesCounter=" << m_writesCounter - << " currBlock=" << write_rep.latest_block); + "ConditionalWrite message handled; writesCounter=" << m_writesCounter << " currBlock=" + << write_rep.latest_block << KVLOG(hasConflict)); return OperationResult::SUCCESS; } @@ -491,26 +532,39 @@ OperationResult InternalCommandsHandler::executeGetBlockDataCommand(const SKVBCG auto block_id = request.block_id; const auto updates = getBlockUpdates(block_id); - if (!updates) { + if (!updates.has_value()) { LOG_WARN(m_logger, "GetBlockData: Failed to retrieve block ID " << block_id); return OperationResult::INTERNAL_ERROR; } - // Each block contains a single metadata key holding the sequence number - const int numMetadataKeys = 1; - auto numOfElements = updates->size() - numMetadataKeys; - LOG_INFO(m_logger, "NUM OF ELEMENTS IN BLOCK = " << numOfElements); - SKVBCReply reply; reply.reply = SKVBCReadReply(); SKVBCReadReply &read_rep = std::get(reply.reply); - read_rep.reads.resize(numOfElements); - size_t i = 0; - for (const auto &[key, value] : *updates) { - if (key != concord::kvbc::IBlockMetadata::kBlockMetadataKeyStr) { - read_rep.reads[i].first.assign(key.begin(), key.end()); - read_rep.reads[i].second.assign(value.begin(), value.end()); - ++i; + + if (auto iter = updates.value().find(s_ignoreBlockStr); iter != updates.value().end()) { + read_rep.reads.resize(1); + const string &ignoreBlockStr = iter->first; + std::vector ignoreBlockAsBytes; + std::transform(ignoreBlockStr.begin(), ignoreBlockStr.end(), std::back_inserter(ignoreBlockAsBytes), [](char c) { + return static_cast(c); + }); + read_rep.reads[0] = {ignoreBlockAsBytes, {}}; + } + + else { + // Each block contains a single metadata key holding the sequence number + const int numMetadataKeys = 1; + const auto numOfElements = updates->size() - numMetadataKeys; + LOG_INFO(m_logger, "NUM OF ELEMENTS IN BLOCK = " << numOfElements); + + read_rep.reads.resize(numOfElements); + size_t i = 0; + for (const auto &[key, value] : *updates) { + if (key != concord::kvbc::IBlockMetadata::kBlockMetadataKeyStr) { + read_rep.reads[i].first.assign(key.begin(), key.end()); + read_rep.reads[i].second.assign(value.begin(), value.end()); + ++i; + } } } diff --git a/tests/simpleKVBC/TesterReplica/internalCommandsHandler.hpp b/tests/simpleKVBC/TesterReplica/internalCommandsHandler.hpp index 5b0ee6a2b7..5e3010a514 100644 --- a/tests/simpleKVBC/TesterReplica/internalCommandsHandler.hpp +++ b/tests/simpleKVBC/TesterReplica/internalCommandsHandler.hpp @@ -68,7 +68,31 @@ class InternalCommandsHandler : public concord::kvbc::ICommandsHandler { std::string &&value, concord::kvbc::categorization::VersionedUpdates &, concord::kvbc::categorization::BlockMerkleUpdates &) const; - + /** + * + * @param requestSize + * @param request + * @param sequenceNum + * @param flags + * @param maxReplySize + * @param outReply + * @param outReplySize + * @param isBlockAccumulationEnabled + * @param blockAccumulatedVerUpdates + * @param blockAccumulatedMerkleUpdates + * @param batchCid - The id of the request batch this request belongs to. + * This information is not available in the execution layer. + * The test client sets it as the Cid of the request to make it so. + * @param requestId - The id of the request old ids are rejected if a request with a higher id + * clientId was perviously executed for a request originating in clientId. + * @param clientId - The id of the client who issued the request, used in conjunction with requestCid to prevent + * the execution of old requests. + * + * @note Batched client requests are expected to reach execution only once, + * in the current implementation, the preprocessor is the only entity aware of client request + * batches and is responsible for satisfying this assumption + * @return + */ bftEngine::OperationResult executeWriteCommand( uint32_t requestSize, const char *request, @@ -79,7 +103,10 @@ class InternalCommandsHandler : public concord::kvbc::ICommandsHandler { uint32_t &outReplySize, bool isBlockAccumulationEnabled, concord::kvbc::categorization::VersionedUpdates &blockAccumulatedVerUpdates, - concord::kvbc::categorization::BlockMerkleUpdates &blockAccumulatedMerkleUpdates); + concord::kvbc::categorization::BlockMerkleUpdates &blockAccumulatedMerkleUpdates, + uint64_t batchCid, + uint64_t requestId, + uint16_t clientId); bftEngine::OperationResult executeReadOnlyCommand(uint32_t requestSize, const char *request, @@ -140,4 +167,11 @@ class InternalCommandsHandler : public concord::kvbc::ICommandsHandler { std::shared_ptr perfManager_; bool m_addAllKeysAsPublic{false}; // Add all key-values in the block merkle category as public ones. concord::kvbc::adapter::ReplicaBlockchain *m_kvbc{nullptr}; + std::unordered_map m_clientToMaxExecutedReqId; + + // This string is used by clients to distinguish blocks that should be ignored by them. + // Some tests expect every block to be created by a request issued by test clients. + // However, internal communication between replicas can also create blocks, e.g: + // When rotating keys. + static constexpr const char* s_ignoreBlockStr = "ignoreBlock"; }; diff --git a/tests/simpleKVBC/TesterReplica/main.cpp b/tests/simpleKVBC/TesterReplica/main.cpp index 9833633fc1..f2785e9f63 100644 --- a/tests/simpleKVBC/TesterReplica/main.cpp +++ b/tests/simpleKVBC/TesterReplica/main.cpp @@ -182,26 +182,29 @@ void run_replica(int argc, char** argv) { } } // namespace concord::kvbc::test +static std::unordered_map defaultHandlers; + namespace { static void signal_handler(int signal_num) { - LOG_INFO(GL, "Program received signal " << signal_num); + LOG_WARN(GL, "Program received signal " << signal_num); + printCallStack(); concord::kvbc::test::timeToExit = true; + defaultHandlers[signal_num](signal_num); } } // namespace int main(int argc, char** argv) { - struct sigaction sa; - LOG_INFO(GL, "skvbc_replia (concord-bft tester replica) starting..."); - memset(&sa, 0, sizeof(sa)); - sa.sa_handler = signal_handler; - sigfillset(&sa.sa_mask); - sigaction(SIGINT, &sa, NULL); - sigaction(SIGTERM, &sa, NULL); + const auto signals = {SIGABRT, SIGILL, SIGFPE, SIGSEGV, SIGTERM, SIGKILL, SIGQUIT, SIGHUP, SIGBUS, SIGSYS, SIGPIPE, SIGSTOP, SIGTSTP, SIGXFSZ}; + + for (int signalCode : signals) { + defaultHandlers[signalCode] = signal(signalCode, signal_handler); + } try { concord::kvbc::test::run_replica(argc, argv); } catch (const std::exception& e) { LOG_FATAL(GL, "exception: " << e.what()); + return 1; } LOG_INFO(GL, "skvbc_replia (concord-bft tester replica) shutting down..."); return 0; diff --git a/tools/KeyfileIOUtils.cpp b/tools/KeyfileIOUtils.cpp index 003803c6f8..3492d6acfc 100644 --- a/tools/KeyfileIOUtils.cpp +++ b/tools/KeyfileIOUtils.cpp @@ -21,6 +21,7 @@ #include "KeyfileIOUtils.hpp" #include "util/yaml_utils.hpp" #include "crypto/openssl/EdDSA.hpp" +#include "util/filesystem.hpp" using concord::crypto::isValidKey; using bftEngine::ReplicaConfig; @@ -79,6 +80,7 @@ static void validateRSAPrivateKey(const std::string& key) { Cryptosystem* inputReplicaKeyfileMultisig(const std::string& filename, ReplicaConfig& config) { using namespace concord::util; + LOG_INFO(GL, "Parsing cryptographic keys configuration file: " << fs::absolute(filename).string()); std::ifstream input(filename); if (!input.is_open()) throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": can't open ") + filename); @@ -110,6 +112,7 @@ Cryptosystem* inputReplicaKeyfileMultisig(const std::string& filename, ReplicaCo } std::cout << "main_key_algorithm=" << mainKeyAlgo << std::endl; + std::cout << "Reading main keys: " << std::endl; config.publicKeysOfReplicas.clear(); for (size_t i = 0; i < config.numReplicas + config.numRoReplicas; ++i) { if ("rsa" == mainKeyAlgo) { @@ -120,7 +123,9 @@ Cryptosystem* inputReplicaKeyfileMultisig(const std::string& filename, ReplicaCo } config.publicKeysOfReplicas.insert(std::pair(i, replicaPublicKeys[i])); } + config.replicaPrivateKey = yaml::readValue(input, "replica_private_key"); + if ("rsa" == mainKeyAlgo) { validateRSAPrivateKey(config.replicaPrivateKey); } else if ("eddsa" == mainKeyAlgo) { @@ -130,9 +135,20 @@ Cryptosystem* inputReplicaKeyfileMultisig(const std::string& filename, ReplicaCo if (config.isReadOnly) return nullptr; + // TODO(yf): protect this code with a feature flag + config.thresholdVerificationKeys_.resize(config.numReplicas); + for (auto& [i, hexKey] : config.publicKeysOfReplicas) { + if (i < config.numReplicas) { + config.thresholdVerificationKeys_[i] = hexKey; + LOG_INFO(GL, KVLOG(i, hexKey)); + } + } + + config.thresholdPrivateKey_ = config.replicaPrivateKey; + return Cryptosystem::fromConfiguration(input, "common", - config.replicaId + 1, + config.replicaId, config.thresholdSystemType_, config.thresholdSystemSubType_, config.thresholdPrivateKey_, From c65e9b0227f7f970abc95df6317646e0f9770196 Mon Sep 17 00:00:00 2001 From: Ynon Flum Date: Wed, 15 Feb 2023 12:39:18 +0200 Subject: [PATCH 04/18] Fix additional apollo tests --- Makefile | 4 +- bftengine/include/bftengine/CryptoManager.hpp | 42 ++- .../include/bftengine/KeyExchangeManager.hpp | 25 +- .../bcstatetransfer/AsyncStateTransferCRE.cpp | 9 +- bftengine/src/bftengine/ClientsManager.cpp | 2 - bftengine/src/bftengine/CryptoManager.cpp | 159 +++++----- .../src/bftengine/KeyExchangeManager.cpp | 113 ++++--- bftengine/src/bftengine/KeyStore.cpp | 2 +- bftengine/src/bftengine/KeyStore.h | 12 +- bftengine/src/bftengine/Reconfiguration.cpp | 3 +- .../src/bftengine/ReplicaForStateTransfer.cpp | 35 +- bftengine/src/bftengine/ReplicaImp.cpp | 76 ++--- bftengine/src/bftengine/ReplicaImp.hpp | 2 +- bftengine/src/bftengine/RequestHandler.cpp | 16 +- bftengine/src/bftengine/SigManager.cpp | 111 ++++--- bftengine/src/bftengine/SigManager.hpp | 7 +- .../src/bftengine/messages/CheckpointMsg.cpp | 3 +- .../tests/SigManager/SigManager_test.cpp | 300 ------------------ .../tests/keyManager/KeyManager_test.cpp | 4 +- .../reconfiguration/src/default_handlers.cpp | 1 - .../src/poll_based_state_client.cpp | 5 +- kvbc/include/categorization/blockchain.h | 3 +- kvbc/src/reconfiguration_kvbc_handler.cpp | 7 +- kvbc/src/v4blockchain/v4_blockchain.cpp | 3 +- libs/communication/StateControl.hpp | 4 +- libs/crypto/openssl/EdDSASigner.hpp | 3 +- libs/crypto/openssl/EdDSAVerifier.hpp | 2 +- .../threshsign/ThresholdSignaturesTypes.cpp | 4 +- .../threshsign/eddsa/EdDSAMultisigSigner.cpp | 4 +- .../eddsa/EdDSAMultisigVerifier.cpp | 10 +- .../threshsign/ThresholdSignaturesTypes.h | 10 +- libs/util/assertUtils.hpp | 45 --- .../TesterReplica/internalCommandsHandler.cpp | 6 +- .../TesterReplica/internalCommandsHandler.hpp | 2 +- tests/simpleKVBC/TesterReplica/main.cpp | 15 +- 35 files changed, 395 insertions(+), 654 deletions(-) diff --git a/Makefile b/Makefile index 2eecd3d666..805fa26c95 100644 --- a/Makefile +++ b/Makefile @@ -185,8 +185,8 @@ BASIC_RUN_PARAMS?=-it --init --rm --privileged=true \ --name="${CONCORD_BFT_DOCKER_CONTAINER}" \ --workdir=${CONCORD_BFT_TARGET_SOURCE_PATH} \ --mount type=bind,source=${CURDIR},target=${CONCORD_BFT_TARGET_SOURCE_PATH}${CONCORD_BFT_CONTAINER_MOUNT_CONSISTENCY} \ - --env APOLLO_RUN_TEST_DIR=${APOLLO_TIMESTAMP} \ - ${CONCORD_BFT_ADDITIONAL_RUN_PARAMS} ${CONCORD_BFT_DOCKER_IMAGE_FULL_PATH} + ${CONCORD_BFT_ADDITIONAL_RUN_PARAMS} \ + ${CONCORD_BFT_DOCKER_IMAGE_FULL_PATH} .DEFAULT_GOAL:=build diff --git a/bftengine/include/bftengine/CryptoManager.hpp b/bftengine/include/bftengine/CryptoManager.hpp index 35a13b9f92..e584e25372 100644 --- a/bftengine/include/bftengine/CryptoManager.hpp +++ b/bftengine/include/bftengine/CryptoManager.hpp @@ -18,12 +18,16 @@ #include "crypto/threshsign/ThresholdSignaturesTypes.h" #include "crypto/threshsign/IThresholdSigner.h" #include "crypto/threshsign/IThresholdVerifier.h" +#include "crypto/threshsign/eddsa/EdDSAMultisigVerifier.h" +#include "crypto/threshsign/eddsa/EdDSAMultisigSigner.h" #include "ReplicaConfig.hpp" #include "IKeyExchanger.hpp" #include "crypto/crypto.hpp" namespace bftEngine { -typedef std::int64_t SeqNum; // TODO [TK] redefinition + +using CheckpointNum = std::uint64_t; +using SeqNum = std::int64_t; // TODO [TK] redefinition constexpr uint16_t checkpointWindowSize = 150; // TODO [TK] redefinition class CryptoManager : public IKeyExchanger, public IMultiSigKeyGenerator { @@ -66,19 +70,31 @@ class CryptoManager : public IKeyExchanger, public IMultiSigKeyGenerator { const std::uint16_t& signerIndex, const SeqNum& sn) override; + /** + * Iterates saved cryptosystems and updates their private key if their public key matches verificationKey + * After ST, if a replica has not executed its key exchange but the network did, the internal private key state needs + * to be synchronized. + * @param secretKey + * @param verificationKey + * @note: Assumes all keys are formatted as hex strings + * @note: TODO: Current implementation is not crash consistent, a ST which was completed after the replica process + * will lose the new private key + */ + void syncPrivateKeyAfterST(const std::string& secretKey, const std::string& verificationKey); + void onCheckpoint(uint64_t newCheckpoint); // Important note: // CryptoManager's cryptosystems are currently implemented using a naive eddsa multisig scheme // The following methods break the abstraction of the threshsign library in order - // to extract ISigner and IVerifier object. + // to extract ISigner and IVerifier objects. // This abstraction is broken to allow using the consensus key as the replica's main key (In SigManager), thus // enabling an operator to change it (key rotation). - // This code will need to be refactored if a different cryptosystem is used. - std::shared_ptr getSigner(SeqNum seq) const; - std::shared_ptr getMultisigVerifier(SeqNum seq) const; - std::array>, 2> getLatestVerifiers() const; - std::array, 2> getLatestSigners() const; + // This code will need to be refactored if a different implementation is used. + std::shared_ptr getSigner(SeqNum seq) const; + std::shared_ptr getMultisigVerifier(SeqNum seq) const; + std::array>, 2> getLatestVerifiers() const; + std::array, 2> getLatestSigners() const; private: /** @@ -101,12 +117,14 @@ class CryptoManager : public IKeyExchanger, public IMultiSigKeyGenerator { void init(); }; - using SeqToSystemMap = std::map>; + using CheckpointToSystemMap = std::map>; // accessing existing Cryptosystems + CheckpointNum getCheckpointOfCryptosystemForSeq(const SeqNum sn) const; std::shared_ptr get(const SeqNum& sn) const; // create CryptoSys for sn if still doesn't exist + // Ensures that there are no more than two cryptosystems std::shared_ptr create(const SeqNum& sn); CryptoManager(std::unique_ptr&& cryptoSys); @@ -117,14 +135,14 @@ class CryptoManager : public IKeyExchanger, public IMultiSigKeyGenerator { CryptoManager& operator=(const CryptoManager&&) = delete; void assertMapSizeValid() const; - const SeqToSystemMap& getSeqToSystem() const; + const CheckpointToSystemMap& checkpointToSystem() const; // chckp -> CryptoSys // TODO: this can be converted to a concurrent queue instead of using a mutex - SeqToSystemMap cryptoSystems_; - // Old cryptosystems can be removed on a checkpoint, which might invalidate + CheckpointToSystemMap cryptoSystems_; + // Old cryptosystems can be removed on a checkpoint/cryptosystem creation which might invalidate // existing cryptoSystems_ iterators. We thus protect cryptoSystems_ access with a mutex - // and rely on shared_ptr to keep old cryptosystems alive in concurrent threads when they lag + // and rely on shared_ptr to keep old cryptosystems alive in concurrent threads mutable std::mutex mutex_; }; } // namespace bftEngine diff --git a/bftengine/include/bftengine/KeyExchangeManager.hpp b/bftengine/include/bftengine/KeyExchangeManager.hpp index 8153d694d3..7d54617856 100644 --- a/bftengine/include/bftengine/KeyExchangeManager.hpp +++ b/bftengine/include/bftengine/KeyExchangeManager.hpp @@ -35,10 +35,19 @@ class KeyExchangeManager { void generateConsensusKeyAndSendInternalClientMsg(const SeqNum& sn); // Send the current main public key of the replica to consensus void sendMainPublicKey(); - // Generates and publish the first replica's key, - void sendInitialKey(const ReplicaImp* repImpInstance, const SeqNum& = 0); - // The execution handler implementation that is called when a key exchange msg has passed consensus. + // Waits for a quorum and calls generateConsensusKeyAndSendInternalClientMsg + void waitForQuorumAndTriggerConsensusExchange(const ReplicaImp* repImpInstance, const SeqNum& = 0); + // The execution handler implementation that is called after a key exchange msg passed consensus. + // The new key pair will be used from two checkpoints after kemsg.generated_sn std::string onKeyExchange(const KeyExchangeMsg& kemsg, const SeqNum& req_sn, const std::string& cid); + /** + * Updates new key pair in both volatile (CryptoManager) and persistent (Reserved pages) memory + * @param repID - The id of the replica which uses pubkey + * @param effective_key_sn - The sequence number from which the new key is to be used + * @param pubkey - The public key of replica repID + * @param cid + */ + void registerNewKeyPair(uint16_t repID, SeqNum effective_key_sn, const std::string& pubkey, const std::string& cid); // Register a IKeyExchanger to notification when keys are rotated. void registerForNotification(IKeyExchanger* ke) { registryToExchange_.push_back(ke); } // Called at the end of state transfer @@ -46,13 +55,7 @@ class KeyExchangeManager { void loadClientPublicKeys(); // whether initial key exchange has occurred - bool exchanged() const { - uint32_t liveClusterSize = ReplicaConfig::instance().waitForFullCommOnStartup ? clusterSize_ : quorumSize_; - bool exchange_self_keys = publicKeys_.keyExists(ReplicaConfig::instance().replicaId); - return ReplicaConfig::instance().getkeyExchangeOnStart() - ? (publicKeys_.numOfExchangedReplicas() + 1 >= liveClusterSize) && exchange_self_keys - : true; - } + bool exchanged() const; const std::string kInitialKeyExchangeCid = "KEY-EXCHANGE-"; const std::string kInitialClientsKeysCid = "CLIENTS-PUB-KEYS-"; ///////// Clients public keys interface/////////////// @@ -72,6 +75,8 @@ class KeyExchangeManager { concord::crypto::KeyFormat, NodeIdType clientId, bool saveToReservedPages); + + std::pair getCandidateKeyPair() const; ///////// end - Clients public keys interface/////////////// std::string getStatus() const; diff --git a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp index 70a2e42e7c..ac5076b34a 100644 --- a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp +++ b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp @@ -113,11 +113,11 @@ class ScalingReplicaHandler : public IStateHandler { if (!configuration_file.good()) { LOG_FATAL(getLogger(), "unable to open the reconfigurations file"); } - configuration_file << (command.config_descriptor + "\n"); - configuration_file.close(); LOG_INFO(getLogger(), "getting new configuration"); bftEngine::ControlStateManager::instance().getNewConfiguration(command.config_descriptor, command.token); bftEngine::ControlStateManager::instance().markRemoveMetadata(); + configuration_file << (command.config_descriptor + "\n"); + configuration_file.close(); LOG_INFO(getLogger(), "completed scaling procedure for " << command.config_descriptor << " restarting the replica"); if (command.restart) bftEngine::ControlStateManager::instance().restart(); return true; @@ -148,8 +148,6 @@ class MainKeyUpdateHandler : public IStateHandler { concord::messages::ClientStateReply crep; concord::messages::deserialize(state.data, crep); concord::messages::ReplicaMainKeyUpdate update = std::get(crep.response); - // TODO(yf): persist key file? - // TODO(yf): update key via cryptomanager bftEngine::CryptoManager::instance().onPublicKeyExchange(update.key, update.sender_id, update.seq_num); return true; } @@ -194,9 +192,8 @@ std::shared_ptr CreFactory::create( } size_t ReplicaCRESigner::signBuffer(const concord::Byte* dataIn, size_t dataLen, concord::Byte* sigOutBuffer) const { - LOG_INFO(GL, "ReplicaCRESigner signing"); + LOG_DEBUG(GL, "ReplicaCRESigner signing"); auto* sigManager = bftEngine::impl::SigManager::instance(); - //return sigManager->getLastReplicaSigner()->signBuffer(dataIn, dataLen, sigOutBuffer); return sigManager->sign(sigManager->getReplicaLastExecutedSeq(), dataIn, dataLen, sigOutBuffer); } diff --git a/bftengine/src/bftengine/ClientsManager.cpp b/bftengine/src/bftengine/ClientsManager.cpp index b44f3e22b0..e1465d6955 100644 --- a/bftengine/src/bftengine/ClientsManager.cpp +++ b/bftengine/src/bftengine/ClientsManager.cpp @@ -148,7 +148,6 @@ void ClientsManager::RepliesInfo::deleteReplyIfNeededSafe(NodeIdType clientId, if (repliesBiMap_.right.find(reqIndex) != repliesBiMap_.right.end()) { savedReplyId = repliesBiMap_.right.at(reqIndex); if (savedReplyId > reqSeqNum) { - printCallStack(); LOG_WARN(CL_MNGR, "A newer reply was already saved" << KVLOG(clientId, reqSeqNum, savedReplyId, reqIndex)); } } @@ -179,7 +178,6 @@ uint16_t ClientsManager::RepliesInfo::getIndex(ReqId reqSeqNum) const { return (replyIt != repliesBiMap_.left.end()) ? replyIt->second : 0; } - /*************************** Class ClientsManager ***************************/ // Initialize: diff --git a/bftengine/src/bftengine/CryptoManager.cpp b/bftengine/src/bftengine/CryptoManager.cpp index d6f3671917..81216c225f 100644 --- a/bftengine/src/bftengine/CryptoManager.cpp +++ b/bftengine/src/bftengine/CryptoManager.cpp @@ -9,10 +9,10 @@ // these subcomponents is subject to the terms and conditions of the sub-component's license, as noted in the LICENSE // file. -#include #include "ReplicaConfig.hpp" #include "Logger.hpp" #include "CryptoManager.hpp" +#include namespace bftEngine { @@ -49,7 +49,7 @@ std::shared_ptr CryptoManager::thresholdVerifierForOptimisti } std::unique_ptr& CryptoManager::getLatestCryptoSystem() const { - return getSeqToSystem().rbegin()->second->cryptosys_; + return checkpointToSystem().rbegin()->second->cryptosys_; } /** @@ -72,6 +72,18 @@ std::tuple Crypto return {priv, pub, getLatestSignatureAlgorithm()}; } +void CryptoManager::syncPrivateKeyAfterST(const std::string& secretKey, const std::string& verificationKey) { + std::lock_guard guard(mutex_); + for (auto& [checkpoint, cryptoSystem] : checkpointToSystem()) { + auto currentKey = cryptoSystem->cryptosys_->getMyVerificationKey(); + LOG_INFO(logger(), "checking keys after ST" << KVLOG(checkpoint, currentKey, secretKey, verificationKey)); + if (currentKey == verificationKey) { + cryptoSystem->cryptosys_->updateKeys(secretKey, verificationKey); + LOG_INFO(logger(), "Updated private key for cryptosystem with checkpoint:" << checkpoint); + } + } +} + void CryptoManager::onPrivateKeyExchange(const std::string& secretKey, const std::string& verificationKey, const SeqNum& sn) { @@ -80,6 +92,7 @@ void CryptoManager::onPrivateKeyExchange(const std::string& secretKey, sys_wrapper->cryptosys_->updateKeys(secretKey, verificationKey); sys_wrapper->init(); } + void CryptoManager::onPublicKeyExchange(const std::string& verificationKey, const std::uint16_t& signerIndex, const SeqNum& sn) { @@ -91,64 +104,55 @@ void CryptoManager::onPublicKeyExchange(const std::string& verificationKey, void CryptoManager::onCheckpoint(uint64_t newCheckpoint) { std::lock_guard guard(mutex_); - std::vector checkpointsToRemove; - for (auto& [checkpoint, _] : getSeqToSystem()) { - if (getSeqToSystem().size() - checkpointsToRemove.size() == 1) { - break; - } - - if (checkpoint + 2 < newCheckpoint) { - checkpointsToRemove.push_back(checkpoint); - } - } - - LOG_INFO(logger(), - "Checkpoint " << newCheckpoint << " reached, removing " << checkpointsToRemove.size() << " stale keys" - << KVLOG(getSeqToSystem().size())); - - if (!checkpointsToRemove.empty()) { - for (auto checkpointToRemove : checkpointsToRemove) { - cryptoSystems_.erase(checkpointToRemove); - LOG_INFO(logger(), - "Removed stale cryptosystem " << KVLOG(checkpointToRemove, newCheckpoint, getSeqToSystem().size())); - assertMapSizeValid(); - // TODO: persist key store - } - } - - // clearOldKeys(); + std::vector checkpointsToRemove; + auto checkpointOfCurrentCryptoSystem = getCheckpointOfCryptosystemForSeq(newCheckpoint * checkpointWindowSize); + + std::experimental::erase_if( + cryptoSystems_, [this, checkpointOfCurrentCryptoSystem, newCheckpoint](const auto& checkpointToSystem) { + auto checkpoint = checkpointToSystem.first; + if (checkpoint < checkpointOfCurrentCryptoSystem) { + LOG_INFO(logger(), "Removing stale cryptosystem " << KVLOG(checkpoint, newCheckpoint, cryptoSystems_.size())); + return true; + } + return false; + }); + assertMapSizeValid(); } -std::shared_ptr CryptoManager::getSigner(SeqNum seq) const { +std::shared_ptr CryptoManager::getSigner(SeqNum seq) const { auto signer = get(seq)->thresholdSigner_; - return signer; + return std::reinterpret_pointer_cast(signer); } -std::shared_ptr CryptoManager::getMultisigVerifier(SeqNum seq) const { +std::shared_ptr CryptoManager::getMultisigVerifier(SeqNum seq) const { auto verifier = get(seq)->thresholdVerifierForOptimisticCommit_; - return verifier; + return std::reinterpret_pointer_cast(verifier); } -std::array>, 2> CryptoManager::getLatestVerifiers() const { +std::array>, 2> CryptoManager::getLatestVerifiers() const { std::lock_guard guard(mutex_); - auto riter = getSeqToSystem().rbegin(); - std::array>, 2> result; - result[0] = {riter->first, riter->second->thresholdVerifierForOptimisticCommit_}; + auto riter = checkpointToSystem().rbegin(); + std::array>, 2> result; + result[0] = { + riter->first, + std::reinterpret_pointer_cast(riter->second->thresholdVerifierForOptimisticCommit_)}; riter++; - if (riter != getSeqToSystem().rend() && riter->second != nullptr) { - result[1] = {riter->first, riter->second->thresholdVerifierForOptimisticCommit_}; + if (riter != checkpointToSystem().rend() && riter->second != nullptr) { + result[1] = { + riter->first, + std::reinterpret_pointer_cast(riter->second->thresholdVerifierForOptimisticCommit_)}; } return result; } -std::array, 2> CryptoManager::getLatestSigners() const { +std::array, 2> CryptoManager::getLatestSigners() const { std::lock_guard guard(mutex_); - auto riter = getSeqToSystem().rbegin(); - std::array, 2> result; - result[0] = riter->second->thresholdSigner_; + auto riter = checkpointToSystem().rbegin(); + std::array, 2> result; + result[0] = std::reinterpret_pointer_cast(riter->second->thresholdSigner_); ++riter; - if (riter != getSeqToSystem().rend() && riter->second != nullptr) { - result[1] = riter->second->thresholdSigner_; + if (riter != checkpointToSystem().rend() && riter->second != nullptr) { + result[1] = std::reinterpret_pointer_cast(riter->second->thresholdSigner_); } return result; } @@ -166,28 +170,35 @@ void CryptoManager::CryptoSystemWrapper::init() { thresholdVerifierForOptimisticCommit_.reset(cryptosys_->createThresholdVerifier(numSigners)); } -std::shared_ptr CryptoManager::get(const SeqNum& sn) const { - std::lock_guard guard(mutex_); +CheckpointNum CryptoManager::getCheckpointOfCryptosystemForSeq(const SeqNum sn) const { // find last chckp that is less than a chckp of a given sn - const uint64_t checkpointUpperBound = (sn - 1) / checkpointWindowSize; - for (auto riter = getSeqToSystem().rbegin(); riter != getSeqToSystem().rend(); ++riter) { + const uint64_t checkpointUpperBound = sn == 0 ? 0 : ((sn - 1) / checkpointWindowSize); + for (auto riter = checkpointToSystem().rbegin(); riter != checkpointToSystem().rend(); ++riter) { auto& [checkpointCandidate, cryptosystem] = *riter; if (checkpointCandidate <= checkpointUpperBound) { - LOG_INFO(logger(), - "Found cryptosystem for " << KVLOG(sn, - checkpointUpperBound, - checkpointCandidate, - cryptosystem, - getSeqToSystem().size(), - checkpointWindowSize)); - return cryptosystem; + LOG_DEBUG(logger(), + "Found cryptosystem for " << KVLOG(sn, + checkpointUpperBound, + checkpointCandidate, + cryptosystem, + checkpointToSystem().size(), + checkpointWindowSize)); + return checkpointCandidate; } } - LOG_FATAL( - logger(), - "Cryptosystem not found " << KVLOG(sn, checkpointUpperBound, getSeqToSystem().size(), checkpointWindowSize)); - ConcordAssert(false && "should never reach here"); + // Replicas might encounter old messages, crashing is not + auto fallbackSystemCheckpoint = checkpointToSystem().rbegin()->first; + LOG_WARN(logger(), + "Cryptosystem not found, returning latest" + << KVLOG(sn, checkpointUpperBound, checkpointToSystem().size(), fallbackSystemCheckpoint)); + return fallbackSystemCheckpoint; + // ConcordAssert(false && "should never reach here"); +} + +std::shared_ptr CryptoManager::get(const SeqNum& sn) const { + std::lock_guard guard(mutex_); + return checkpointToSystem().at(getCheckpointOfCryptosystemForSeq(sn)); } std::shared_ptr CryptoManager::create(const SeqNum& sn) { @@ -195,20 +206,21 @@ std::shared_ptr CryptoManager::create(const assertMapSizeValid(); // Cryptosystem for this sn will be activated upon reaching a second checkpoint from now uint64_t chckp = (sn / checkpointWindowSize) + 2; - if (auto it = getSeqToSystem().find(chckp); it != getSeqToSystem().end()) return it->second; + if (auto it = checkpointToSystem().find(chckp); it != checkpointToSystem().end()) { + LOG_WARN(logger(), "Cryptosystem already exists for checkpoint " << chckp); + return it->second; + } + // copy construct new Cryptosystem from a last one as we want it to include all the existing keys std::unique_ptr cs = - std::make_unique(*getSeqToSystem().rbegin()->second->cryptosys_.get()); + std::make_unique(*checkpointToSystem().rbegin()->second->cryptosys_.get()); - while (cryptoSystems_.size() > 2) { - auto currentSystem = cryptoSystems_.begin(); - cryptoSystems_.erase(currentSystem); - LOG_INFO(logger(), "Removed stale cryptosystem with checkpoint " << currentSystem->first); - } - - auto insert_result = cryptoSystems_.insert(std::make_pair(chckp, std::make_shared(std::move(cs)))); + auto insert_result = + cryptoSystems_.insert(std::make_pair(chckp, std::make_shared(std::move(cs)))); auto ret = insert_result.first->second; - LOG_INFO(logger(), "created new cryptosystem for checkpoint: " << chckp << ", insertion success: " << insert_result.second); + LOG_INFO(logger(), + "Created new cryptosystem for checkpoint: " << chckp << ", insertion success: " << insert_result.second + << KVLOG(cryptoSystems_.size())); assertMapSizeValid(); return ret; } @@ -224,12 +236,9 @@ logging::Logger& CryptoManager::logger() const { return logger_; } -void CryptoManager::assertMapSizeValid() const { - ConcordAssertGE(cryptoSystems_.size(), 1); - ConcordAssertLE(cryptoSystems_.size(), 3); -} +void CryptoManager::assertMapSizeValid() const { ConcordAssertGE(cryptoSystems_.size(), 1); } -const CryptoManager::SeqToSystemMap& CryptoManager::getSeqToSystem() const { +const CryptoManager::CheckpointToSystemMap& CryptoManager::checkpointToSystem() const { assertMapSizeValid(); return cryptoSystems_; } diff --git a/bftengine/src/bftengine/KeyExchangeManager.cpp b/bftengine/src/bftengine/KeyExchangeManager.cpp index 00ade0268c..a91b996de3 100644 --- a/bftengine/src/bftengine/KeyExchangeManager.cpp +++ b/bftengine/src/bftengine/KeyExchangeManager.cpp @@ -72,12 +72,39 @@ std::string KeyExchangeManager::generateCid(std::string cid) { return cid; } +void KeyExchangeManager::registerNewKeyPair(uint16_t repID, + SeqNum sn, + const std::string& pubkey, + const std::string& cid) { + const bool isSelfKeyExchange = repID == repID_; + publicKeys_.push(repID, pubkey, sn); + if (!isSelfKeyExchange) { + for (auto* e : registryToExchange_) e->onPublicKeyExchange(pubkey, repID, sn); + metrics_->public_key_exchange_for_peer_counter++; + return; + } + + private_keys_.key_data().generated = seq_candidate_map_[sn].generated; + seq_candidate_map_[sn].generated.clear(); + candidate_private_keys_.generated.clear(); + // erasing seqnum from the map + seq_candidate_map_.erase(sn); + ConcordAssert(private_keys_.key_data().generated.pub == kemsg.pubkey); + private_keys_.onKeyExchange(cid, sn); + for (auto e : registryToExchange_) e->onPrivateKeyExchange(private_keys_.key_data().keys[sn], kemsg.pubkey, sn); + metrics_->self_key_exchange_counter++; +} + std::string KeyExchangeManager::onKeyExchange(const KeyExchangeMsg& kemsg, const SeqNum& req_sn, const std::string& cid) { - const auto& sn = kemsg.generated_sn; - SCOPED_MDC_SEQ_NUM(std::to_string(sn)); - LOG_INFO(KEY_EX_LOG, kemsg.toString() << KVLOG(sn, cid, exchanged())); + const bool isSelfKeyExchange = kemsg.repID == repID_; + bool sentKeyUpdateToExecution = false; + SCOPED_MDC_SEQ_NUM(std::to_string(kemsg.generated_sn)); + // The key will be used from this sequence forwards + // new keys are used two checkpoints after reaching consensus + const SeqNum sn = kemsg.generated_sn; + LOG_INFO(KEY_EX_LOG, KVLOG(kemsg.toString(), sn, cid, exchanged())); // client query if (kemsg.op == KeyExchangeMsg::HAS_KEYS) { LOG_INFO(KEY_EX_LOG, "Has keys: " << std::boolalpha << exchanged() << std::noboolalpha); @@ -90,42 +117,34 @@ std::string KeyExchangeManager::onKeyExchange(const KeyExchangeMsg& kemsg, return "invalid epoch"; } // reject key exchange message if generated seq_num is outside working window - if (sn < ((static_cast(req_sn) / kWorkWindowSize) * kWorkWindowSize)) { + if (sn < ((req_sn / kWorkWindowSize) * kWorkWindowSize)) { LOG_WARN(KEY_EX_LOG, "Generated sequence number is outside working window, ignore..." << KVLOG(sn, req_sn)); return "gen_seq_num_ouside_workingwindow"; } if (publicKeys_.keyExists(kemsg.repID, sn)) return "ok"; - publicKeys_.push(kemsg, sn); - if (kemsg.repID == repID_ && seq_candidate_map_.count(sn)) { // initiated by me - private_keys_.key_data().generated = seq_candidate_map_[sn].generated; - seq_candidate_map_[sn].generated.clear(); - candidate_private_keys_.generated.clear(); - // erasing seqnum from the map - seq_candidate_map_.erase(sn); - ConcordAssert(private_keys_.key_data().generated.pub == kemsg.pubkey); - private_keys_.onKeyExchange(cid, sn); - for (auto e : registryToExchange_) e->onPrivateKeyExchange(private_keys_.key_data().keys[sn], kemsg.pubkey, sn); - metrics_->self_key_exchange_counter++; - } else { // initiated by others - for (auto e : registryToExchange_) e->onPublicKeyExchange(kemsg.pubkey, kemsg.repID, sn); - metrics_->public_key_exchange_for_peer_counter++; - } + + registerNewKeyPair(kemsg.repID, sn, kemsg.pubkey, cid); + auto liveQuorumSize = ReplicaConfig::instance().waitForFullCommOnStartup ? clusterSize_ : quorumSize_; if (ReplicaConfig::instance().getkeyExchangeOnStart() && (publicKeys_.numOfExchangedReplicas() <= liveQuorumSize)) LOG_INFO(KEY_EX_LOG, "Exchanged [" << publicKeys_.numOfExchangedReplicas() << "] out of [" << liveQuorumSize << "]" - << KVLOG(kemsg.repID, - initial_exchange_, - exchanged(), - ReplicaConfig::instance().publishReplicasMasterKeyOnStartup)); + << KVLOG(kemsg.repID, initial_exchange_, exchanged())); if (!initial_exchange_ && exchanged()) { initial_exchange_ = true; - if (ReplicaConfig::instance().getkeyExchangeOnStart() - /*&&ReplicaConfig::instance().publishReplicasMasterKeyOnStartup*/ - ) { + if (ReplicaConfig::instance().getkeyExchangeOnStart() && + ReplicaConfig::instance().publishReplicasMasterKeyOnStartup) { sendMainPublicKey(); + sentKeyUpdateToExecution = true; } } + + // The key was exchanged successfully, write a block containing the new key to the chain + // TODO: need to persist key state and new block atomically + // (Cannot be done with current reserved pages implementation) + if (ReplicaConfig::instance().singleSignatureScheme && isSelfKeyExchange && !sentKeyUpdateToExecution) { + sendMainPublicKey(); + } return "ok"; } @@ -154,7 +173,7 @@ void KeyExchangeManager::loadPublicKeys() { if (ReplicaConfig::instance().getkeyExchangeOnStart() && exchanged()) { ConcordAssertGE(num_loaded, liveQuorumSize); } - LOG_INFO(KEY_EX_LOG, "building crypto system after state transfer" << KVLOG(num_loaded)); + LOG_INFO(KEY_EX_LOG, "rebuilding crypto system after state transfer" << KVLOG(num_loaded)); notifyRegistry(); } @@ -235,22 +254,18 @@ void KeyExchangeManager::sendMainPublicKey() { latestPublicKey, "hex", static_cast(SigManager::instance()->getMainKeyAlgorithm()), - static_cast(seq)}; + static_cast(seq) * checkpointWindowSize}; // Mark this request as an internal one std::vector data_vec; concord::messages::serialize(data_vec, req); - std::string sig(SigManager::instance()->getMySigLength(), '\0'); - SigManager::instance()->sign(SigManager::instance()->getReplicaLastExecutedSeq(), - reinterpret_cast(data_vec.data()), - data_vec.size(), - sig.data()); - req.signature = std::vector(sig.begin(), sig.end()); + req.signature.resize(SigManager::instance()->getMySigLength()); + SigManager::instance()->sign( + SigManager::instance()->getReplicaLastExecutedSeq(), data_vec.data(), data_vec.size(), req.signature.data()); data_vec.clear(); concord::messages::serialize(data_vec, req); - std::string strMsg(data_vec.begin(), data_vec.end()); std::string cid = "ReplicaMainKeyUpdate_" + std::to_string(repID_); - client_->sendRequest(RECONFIG_FLAG, strMsg.size(), strMsg.c_str(), cid); - LOG_INFO(KEY_EX_LOG, "Replica has published its main public key to the consensus"); + client_->sendRequest(RECONFIG_FLAG, data_vec.size(), reinterpret_cast(data_vec.data()), cid); + LOG_INFO(KEY_EX_LOG, cid + " sent to reconfiguration engine"); } void KeyExchangeManager::generateConsensusKeyAndSendInternalClientMsg(const SeqNum& sn) { @@ -288,9 +303,8 @@ void KeyExchangeManager::generateConsensusKeyAndSendInternalClientMsg(const SeqN msg.pubkey = candidate.pub; msg.algorithm = candidate.algorithm; - LOG_INFO(KEY_EX_LOG, "Sending consensus key exchange :" << KVLOG(candidate.cid, msg.pubkey, msg.algorithm)); + LOG_INFO(KEY_EX_LOG, "Sending consensus key exchange :" << KVLOG(sn, candidate.cid, msg.pubkey, msg.algorithm)); client_->sendRequest(bftEngine::KEY_EXCHANGE_FLAG, msg, candidate.cid); - sendMainPublicKey(); metrics_->sent_key_exchange_counter++; } @@ -341,7 +355,7 @@ void KeyExchangeManager::loadClientPublicKey(const std::string& key, if (saveToReservedPages) saveClientsPublicKeys(SigManager::instance()->getClientsPublicKeys()); } -void KeyExchangeManager::sendInitialKey(const ReplicaImp* repImpInstance, const SeqNum& s) { +void KeyExchangeManager::waitForQuorumAndTriggerConsensusExchange(const ReplicaImp* repImpInstance, const SeqNum& s) { std::unique_lock lock(startup_mutex_); SCOPED_MDC(MDC_REPLICA_ID_KEY, std::to_string(ReplicaConfig::instance().replicaId)); if (!ReplicaConfig::instance().waitForFullCommOnStartup) { @@ -449,4 +463,23 @@ bool KeyExchangeManager::PrivateKeys::load() { return true; } +bool KeyExchangeManager::exchanged() const { + uint32_t liveClusterSize = ReplicaConfig::instance().waitForFullCommOnStartup ? clusterSize_ : quorumSize_; + bool exchange_self_keys = publicKeys_.keyExists(ReplicaConfig::instance().replicaId); + LOG_DEBUG(KEY_EX_LOG, + KVLOG(ReplicaConfig::instance().waitForFullCommOnStartup, + clusterSize_, + quorumSize_, + exchange_self_keys, + ReplicaConfig::instance().getkeyExchangeOnStart(), + publicKeys_.numOfExchangedReplicas())); + return ReplicaConfig::instance().getkeyExchangeOnStart() + ? (publicKeys_.numOfExchangedReplicas() + 1 >= liveClusterSize) && exchange_self_keys + : true; +} +std::pair KeyExchangeManager::getCandidateKeyPair() const { + auto& candidate = candidate_private_keys_.generated; + return {candidate.priv, candidate.pub}; +} + } // namespace bftEngine::impl diff --git a/bftengine/src/bftengine/KeyStore.cpp b/bftengine/src/bftengine/KeyStore.cpp index 7a088da772..31673c3255 100644 --- a/bftengine/src/bftengine/KeyStore.cpp +++ b/bftengine/src/bftengine/KeyStore.cpp @@ -39,7 +39,7 @@ std::optional ClusterKeyStore::loadReplicaKeyStoreF std::istringstream iss(buffer_); PublicKeys ks; PublicKeys::deserialize(iss, ks); - //std::vector keysToRemove; + // std::vector keysToRemove; for (auto [sn, pk] : ks.keys) { /*if (ks.keys.size() - keysToRemove.size() > 2) { keysToRemove.push_back(sn); diff --git a/bftengine/src/bftengine/KeyStore.h b/bftengine/src/bftengine/KeyStore.h index a188f9b01c..d72c81fef3 100644 --- a/bftengine/src/bftengine/KeyStore.h +++ b/bftengine/src/bftengine/KeyStore.h @@ -35,7 +35,7 @@ class ClusterKeyStore : public ResPagesClient { auto res = keys.insert(std::make_pair(sn, pub)); if (!res.second) ConcordAssert(pub == res.first->second) // if existed expect same key } - void remove(const SeqNum sn) { ConcordAssertEQ(keys.erase(sn), 1);} + void remove(const SeqNum sn) { ConcordAssertEQ(keys.erase(sn), 1); } void serializeDataMembers(std::ostream& outStream) const { serialize(outStream, keys); } void deserializeDataMembers(std::istream& inStream) { deserialize(inStream, keys); } std::map keys; @@ -47,10 +47,10 @@ class ClusterKeyStore : public ResPagesClient { loadAllReplicasKeyStoresFromReservedPages(); } - void push(const KeyExchangeMsg& kem, const SeqNum& sn) { - LOG_INFO(KEY_EX_LOG, kem.toString() << " seqnum: " << sn); - clusterKeys_[kem.repID].push(kem.pubkey, sn); - saveReplicaKeyStoreToReserevedPages(kem.repID); + void push(const uint16_t repID, const std::string& pubkey, const SeqNum& sn) { + LOG_INFO(KEY_EX_LOG, "Pushing public key" << KVLOG(repID, sn)); + clusterKeys_[repID].push(pubkey, sn); + saveReplicaKeyStoreToReserevedPages(repID); log(); } @@ -81,7 +81,7 @@ class ClusterKeyStore : public ResPagesClient { } void log() const { - LOG_INFO(KEY_EX_LOG, "Cluster Public Keys (size " << clusterSize_ << "):"); + LOG_INFO(KEY_EX_LOG, "Cluster Public Keys in reserved pages (Cluster size " << clusterSize_ << "):"); for (auto [repid, PKs] : clusterKeys_) for (auto [sn, pubkey] : PKs.keys) LOG_INFO(KEY_EX_LOG, "repId:" << repid << "\tseqnum: " << sn << "\tpubkey: " << pubkey); diff --git a/bftengine/src/bftengine/Reconfiguration.cpp b/bftengine/src/bftengine/Reconfiguration.cpp index 420523751d..eb520dfd33 100644 --- a/bftengine/src/bftengine/Reconfiguration.cpp +++ b/bftengine/src/bftengine/Reconfiguration.cpp @@ -33,7 +33,8 @@ bool ReconfigurationHandler::handle(const WedgeCommand& cmd, uint32_t, const std::optional&, concord::messages::ReconfigurationResponse&) { - LOG_INFO(getLogger(), "Wedge command instructs replica to stop at next checkpoint after sequence number " << bft_seq_num); + LOG_INFO(getLogger(), + "Wedge command instructs replica to stop at next checkpoint after sequence number " << bft_seq_num); bftEngine::ControlStateManager::instance().setStopAtNextCheckpoint(bft_seq_num); if (cmd.noop == false) addCreateDbSnapshotCbOnWedge(true); diff --git a/bftengine/src/bftengine/ReplicaForStateTransfer.cpp b/bftengine/src/bftengine/ReplicaForStateTransfer.cpp index 0d1dbfe229..7fe7ee2767 100644 --- a/bftengine/src/bftengine/ReplicaForStateTransfer.cpp +++ b/bftengine/src/bftengine/ReplicaForStateTransfer.cpp @@ -25,6 +25,7 @@ #include "bcstatetransfer/AsyncStateTransferCRE.hpp" #include "client/reconfiguration/poll_based_state_client.hpp" #include "KeyExchangeManager.hpp" +#include "CryptoManager.hpp" #include "SigManager.hpp" namespace bftEngine::impl { @@ -81,22 +82,35 @@ void ReplicaForStateTransfer::start() { msgsCommunicator_, msgHandlers_, std::make_unique()); stateTransfer->setReconfigurationEngine(cre_); stateTransfer->addOnTransferringCompleteCallback( - [this](std::uint64_t sequence_number) { + [this](std::uint64_t checkpoint) { // TODO - The next lines up to comment 'YYY' do not belong here (CRE) - consider refactor or move outside if (!config_.isReadOnly) { // Load the public keys of the other replicas from reserved pages // so that their responses can be validated + cre_->halt(); KeyExchangeManager::instance().loadPublicKeys(); + // Need to update private key to match the loaded public key in case they differ (key exchange was executed + // on other replicas but not on this one, finishing ST does not mean that missed key exchanges are executed) + // This can be done by iterating the saved cryptosystems and updating their private key if their + // public key matches the candidate saved in KeyExchangeManager + + // Clear old keys + CryptoManager::instance().onCheckpoint(checkpoint); + auto [priv, pub] = KeyExchangeManager::instance().getCandidateKeyPair(); + CryptoManager::instance().syncPrivateKeyAfterST(priv, pub); + // Make sure to sign the reconfiguration client messages using the key // other replicas expect - SigManager::instance()->setReplicaLastExecutedSeq(sequence_number * checkpointWindowSize); + SigManager::instance()->setReplicaLastExecutedSeq(checkpoint * checkpointWindowSize); + cre_->resume(); // At this point, we, if are not going to have another blocks in state transfer. So, we can safely stop CRE. // if there is a reconfiguration state change that prevents us from starting another state transfer (i.e. // scaling) then CRE probably won't work as well. // 1. First, make sure we handled the most recent available updates. - concord::client::reconfiguration::PollBasedStateClient *pbc = - (concord::client::reconfiguration::PollBasedStateClient *)(cre_->getStateClient()); + auto *pbc = + reinterpret_cast(cre_->getStateClient()); + bool succ = false; while (!succ) { auto latestHandledUpdate = cre_->getLatestKnownUpdateBlock(); @@ -109,12 +123,11 @@ void ReplicaForStateTransfer::start() { succ = false; break; } // else if (!isGettingBlocks) + // 2. Now we can safely halt cre. We know for sure that there are no update in the state transferred + // blocks that haven't been handled yet + cre_->halt(); } - } // while (!succ) { - LOG_INFO(GL, "halting cre"); - // 2. Now we can safely halt cre. We know for sure that there are no update in the state transffered - // blocks that haven't been handled yet - cre_->halt(); + } // while (!succ) } }, IStateTransfer::StateTransferCallBacksPriorities::HIGH); @@ -180,8 +193,6 @@ Timers::Handle ReplicaForStateTransfer::addOneShotTimer(uint32_t timeoutMilli) { [this](concordUtil::Timers::Handle h) { stateTransfer->onTimer(); }); } -void ReplicaForStateTransfer::resumeCRE() { - cre_->resume(); -} +void ReplicaForStateTransfer::resumeCRE() { cre_->resume(); } } // namespace bftEngine::impl diff --git a/bftengine/src/bftengine/ReplicaImp.cpp b/bftengine/src/bftengine/ReplicaImp.cpp index 30187273b3..5d130e8f87 100644 --- a/bftengine/src/bftengine/ReplicaImp.cpp +++ b/bftengine/src/bftengine/ReplicaImp.cpp @@ -260,21 +260,15 @@ void ReplicaImp::validatedMessageHandler(CarrierMesssage *msg) { } } -/** - * validateMessage This is synchronous validate message. - * - * @param msg : Message that can validate itself as quick as possible. - * @return : returns true if message validation succeeds else return false. - */ -bool ReplicaImp::validateMessage(MessageBase *msg) { +template +bool ReplicaImp::validateMessage(MessageType *msg) { if (config_.debugStatisticsEnabled) { DebugStatistics::onReceivedExMessage(msg->type()); } try { if constexpr (std::is_same_v) { msg->validate(*repsInfo, false); - } - else { + } else { msg->validate(*repsInfo); } return true; @@ -282,27 +276,8 @@ bool ReplicaImp::validateMessage(MessageBase *msg) { onReportAboutInvalidMessage(msg, e.what()); return false; } - -} - -/** - * validateMessage This is synchronous validate message. - * - * @param msg : Message that can validate itself as quick as possible.. - * @return : returns true if message validation succeeds else return false. - */ -/*bool ReplicaImp::validateMessage(MessageBase *msg) { - auto result = validateMessageGeneric(msg, [msg, this](){msg->validate(*repsInfo);}); - LOG_INFO(GL, "Validated generic message" << KVLOG(result)); - return result; } -bool ReplicaImp::validateMessage(CheckpointMsg *msg) { - auto result = validateMessageGeneric(msg, [msg, this](){msg->validate(*repsInfo, false);}); - LOG_INFO(GL, "Validated checkpoint message" << KVLOG(result)); - return result; -}*/ - /** * asyncValidateMessage This is a family of asynchronous message which just schedules * the validation in a thread bag and returns peacefully. This will also translate the message @@ -321,14 +296,14 @@ void ReplicaImp::asyncValidateMessage(std::unique_ptr msg) { threadPool.async( [this](auto *unValidatedMsg, auto *replicaInfo, auto *incomingMessageQueue) { - UNUSED(this); - if (!validateMessage(unValidatedMsg)) { - delete unValidatedMsg; - return; - } + UNUSED(this); + if (!validateMessage(unValidatedMsg)) { + delete unValidatedMsg; + return; + } - CarrierMesssage *validatedCarrierMsg = new ValidatedMessageCarrierInternalMsg(unValidatedMsg); - incomingMessageQueue->pushInternalMsg(validatedCarrierMsg); + CarrierMesssage *validatedCarrierMsg = new ValidatedMessageCarrierInternalMsg(unValidatedMsg); + incomingMessageQueue->pushInternalMsg(validatedCarrierMsg); }, msg.release(), repsInfo, @@ -620,9 +595,12 @@ bool ReplicaImp::checkSendPrePrepareMsgPrerequisites() { LOG_INFO(GL, "Will not send PrePrepare since next sequence number [" << primaryLastUsedSeqNum + numOfTransientPreprepareMsgs_ + 1 << "] exceeds concurrency threshold [" - << lastExecutedSeqNum + config_.getconcurrencyLevel() + activeExecutions_ << "]" << - KVLOG(primaryLastUsedSeqNum, numOfTransientPreprepareMsgs_, lastExecutedSeqNum, - config_.getconcurrencyLevel(), activeExecutions_)); + << lastExecutedSeqNum + config_.getconcurrencyLevel() + activeExecutions_ << "]" + << KVLOG(primaryLastUsedSeqNum, + numOfTransientPreprepareMsgs_, + lastExecutedSeqNum, + config_.getconcurrencyLevel(), + activeExecutions_)); return false; } metric_concurrency_level_.Get().Set(primaryLastUsedSeqNum + numOfTransientPreprepareMsgs_ + 1 - lastExecutedSeqNum); @@ -4438,7 +4416,7 @@ ReplicaImp::ReplicaImp(bool firstTime, onViewNumCallbacks_.add([&](bool) { if (config_.keyExchangeOnStart && !KeyExchangeManager::instance().exchanged()) { LOG_INFO(GL, "key exchange has not been finished yet. Give it another try"); - KeyExchangeManager::instance().sendInitialKey(this); + KeyExchangeManager::instance().waitForQuorumAndTriggerConsensusExchange(this); } }); stateTransfer->addOnFetchingStateChangeCallback([&](uint64_t) { @@ -4447,8 +4425,8 @@ ReplicaImp::ReplicaImp(bool firstTime, // initial key exchange after completing ST if (!isCollectingState()) { if (ReplicaConfig::instance().getkeyExchangeOnStart() && !KeyExchangeManager::instance().exchanged()) { - KeyExchangeManager::instance().sendInitialKey(this, lastExecutedSeqNum); - LOG_INFO(GL, "Send initial key exchange after completing state transfer " << KVLOG(lastExecutedSeqNum)); + KeyExchangeManager::instance().waitForQuorumAndTriggerConsensusExchange(this, lastExecutedSeqNum); + LOG_INFO(GL, "Send key exchange after completing state transfer " << KVLOG(lastExecutedSeqNum)); } } else { LOG_WARN(GL, "State Transfer is still collecting! Initial key exchange cannot yet be initiated!"); @@ -4682,10 +4660,11 @@ void ReplicaImp::start() { msgsCommunicator_->startMsgsProcessing(config_.getreplicaId()); if (ReplicaConfig::instance().getkeyExchangeOnStart() && !KeyExchangeManager::instance().exchanged()) { - KeyExchangeManager::instance().sendInitialKey(this); + KeyExchangeManager::instance().waitForQuorumAndTriggerConsensusExchange(this); } else { - // If key exchange is disabled, first publish the replica's main (rsa/eddsa) key to clients - if (ReplicaConfig::instance().publishReplicasMasterKeyOnStartup) { + // If key exchange is disabled, first publish the replica's main key to clients + if (ReplicaConfig::instance().singleSignatureScheme || + ReplicaConfig::instance().publishReplicasMasterKeyOnStartup) { KeyExchangeManager::instance().sendMainPublicKey(); } } @@ -5104,7 +5083,7 @@ void ReplicaImp::executeRequests(PrePrepareMsg *ppMsg, Bitmap &requestSet, Times reqIdx++; ClientRequestMsg req((ClientRequestMsgHeader *)requestBody); - LOG_INFO(GL, "Iterating request" << KVLOG(req.senderId(), req.requestSeqNum())); + LOG_DEBUG(GL, "Iterating request" << KVLOG(req.senderId(), req.requestSeqNum())); if (!requestSet.get(tmp) || req.requestLength() == 0) { InternalMessage im = RemovePendingForExecutionRequest{req.clientProxyId(), req.requestSeqNum()}; @@ -5872,23 +5851,19 @@ void ReplicaImp::primaryPushNoOpIfWedgePending(SeqNum seq) { if (seqNumToStopAt.has_value() && seqNumToStopAt.value() > lastExecutedSeqNum && isCurrentPrimary()) { // If after execution, we discover that we need to wedge at some futuer point, push a noop command to the incoming // messages queue. - LOG_INFO(GL, "sending noop command to bring the system into wedge checkpoint"); + LOG_INFO(GL, "sending noop command to bring the system into wedge checkpoint seq:" << seqNumToStopAt.value()); concord::messages::ReconfigurationRequest req; req.sender = config_.replicaId; req.command = concord::messages::WedgeCommand{config_.replicaId, true}; // Mark this request as an internal one - //auto siglength = 64; std::vector data_vec; concord::messages::serialize(data_vec, req); - LOG_INFO(GL, "wat-1" << KVLOG(data_vec.size(), SigManager::instance()->getMySigLength(), req.signature.size())); req.signature.resize(SigManager::instance()->getMySigLength()); - LOG_INFO(GL, "wat0" << KVLOG(data_vec.size(), SigManager::instance()->getMySigLength(), req.signature.size())); SigManager::instance()->sign( SigManager::instance()->getReplicaLastExecutedSeq(), data_vec.data(), data_vec.size(), req.signature.data()); data_vec.clear(); concord::messages::serialize(data_vec, req); - LOG_INFO(GL, "wat1" << KVLOG(data_vec.size(), req.signature.size())); auto requestSeqNum = std::chrono::duration_cast(getMonotonicTime().time_since_epoch()).count(); auto crm = new ClientRequestMsg(internalBFTClient_->getClientId(), @@ -5898,7 +5873,6 @@ void ReplicaImp::primaryPushNoOpIfWedgePending(SeqNum seq) { reinterpret_cast(data_vec.data()), 60000, "wedge-noop-command-" + std::to_string(seq)); - LOG_INFO(GL, "wat2" << KVLOG(data_vec.size(), req.signature.size())); // Now, try to send a new PrePrepare message immediately, without waiting to a new batch onMessage(std::make_unique(crm)); tryToSendPrePrepareMsg(false); diff --git a/bftengine/src/bftengine/ReplicaImp.hpp b/bftengine/src/bftengine/ReplicaImp.hpp index 825e4d7374..d8b69e1165 100644 --- a/bftengine/src/bftengine/ReplicaImp.hpp +++ b/bftengine/src/bftengine/ReplicaImp.hpp @@ -372,7 +372,7 @@ class ReplicaImp : public InternalReplicaApi, public ReplicaForStateTransfer { void recoverRequests(); template - bool validateMessage(MessageType *msg); + bool validateMessage(MessageType* msg); std::function getMessageValidator(); diff --git a/bftengine/src/bftengine/RequestHandler.cpp b/bftengine/src/bftengine/RequestHandler.cpp index fa47543c3b..c4d759075f 100644 --- a/bftengine/src/bftengine/RequestHandler.cpp +++ b/bftengine/src/bftengine/RequestHandler.cpp @@ -56,11 +56,15 @@ void RequestHandler::execute(IRequestsHandler::ExecutionRequestsQueue& requests, concordUtils::SpanWrapper& parent_span) { bool has_pruning_request = false; for (auto& req : requests) { - LOG_INFO(GL, "Executing request: " << - KVLOG(req.clientId, req.cid, req.flags, req.requestSequenceNum, - req.signature.size(), req.executionSequenceNum, req.requestSize)); + LOG_INFO(GL, + "Executing request: " << KVLOG(req.clientId, + req.cid, + req.flags, + req.requestSequenceNum, + req.signature.size(), + req.executionSequenceNum, + req.requestSize)); if (req.flags & KEY_EXCHANGE_FLAG) { - //TODO(yf): maybe add the mainkeyupdate block here KeyExchangeMsg ke = KeyExchangeMsg::deserializeMsg(req.request, req.requestSize); LOG_INFO(KEY_EX_LOG, "BFT handler received KEY_EXCHANGE msg " << ke.toString()); auto resp = impl::KeyExchangeManager::instance().onKeyExchange(ke, req.executionSequenceNum, req.cid); @@ -76,7 +80,9 @@ void RequestHandler::execute(IRequestsHandler::ExecutionRequestsQueue& requests, } else if (req.flags & MsgFlag::RECONFIG_FLAG) { ReconfigurationRequest rreq; deserialize(std::vector(req.request, req.request + req.requestSize), rreq); - LOG_INFO(GL, "Executing Reconfig request: " << KVLOG(rreq.signature.size(), rreq.command.index(), rreq.sender, rreq.id)); + LOG_INFO( + GL, + "Executing Reconfig request: " << KVLOG(rreq.signature.size(), rreq.command.index(), rreq.sender, rreq.id)); has_pruning_request = std::holds_alternative(rreq.command); ReconfigurationResponse rsi_res = reconfig_dispatcher_.dispatch(rreq, req.executionSequenceNum, timestamp); // in case of read request return only a success part of and replica specific info in the response diff --git a/bftengine/src/bftengine/SigManager.cpp b/bftengine/src/bftengine/SigManager.cpp index 1a5c08cf2d..2860008199 100644 --- a/bftengine/src/bftengine/SigManager.cpp +++ b/bftengine/src/bftengine/SigManager.cpp @@ -201,7 +201,6 @@ uint16_t SigManager::getSigLength(PrincipalId pid) const { std::shared_lock lock(mutex_); if (auto pos = verifiers_.find(pid); pos != verifiers_.end()) { auto result = pos->second->signatureLength(); - // LOG_INFO(GL, "Sig size for id: " << pid << "is " << result); return result; } else { LOG_ERROR(GL, "Unrecognized pid " << pid); @@ -212,7 +211,7 @@ uint16_t SigManager::getSigLength(PrincipalId pid) const { bool SigManager::verifyNonReplicaSig( PrincipalId pid, const concord::Byte* data, size_t dataLength, const concord::Byte* sig, uint16_t sigLength) const { - LOG_INFO(GL, "Validating NON-replica with: " << KVLOG(myId_, pid, sigLength)); + LOG_DEBUG(GL, "Validating NON-replica with: " << KVLOG(myId_, pid, sigLength)); bool result = false; { std::shared_lock lock(mutex_); @@ -251,18 +250,27 @@ bool SigManager::verifyNonReplicaSig( metrics_component_.UpdateAggregator(); } } - LOG_INFO(GL, - "NON-replica validation result: " << KVLOG( - result, myId_, pid, sigLength, idOfReadOnlyReplica, idOfExternalClient)); + LOG_DEBUG(GL, + "NON-replica validation result: " << KVLOG( + result, myId_, pid, sigLength, idOfReadOnlyReplica, idOfExternalClient)); return result; } size_t SigManager::sign(SeqNum seq, const concord::Byte* data, size_t dataLength, concord::Byte* outSig) const { ConcordAssert(replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica()); - auto signer = CryptoManager::instance().getSigner(seq); - auto& rawSigner = *reinterpret_cast(signer.get()); - auto result = rawSigner.signBuffer(data, dataLength, outSig); - LOG_INFO(GL, "Signing as replica with " << KVLOG(myId_, seq, rawSigner.getPrivKey(), rawSigner.signatureLength(), result)); + concord::crypto::ISigner* rawSigner = nullptr; + std::shared_ptr signer; + if (ReplicaConfig::instance().singleSignatureScheme) { + signer = CryptoManager::instance().getSigner(seq); + rawSigner = signer.get(); + } else { + rawSigner = mySigner_.get(); + } + auto result = rawSigner->signBuffer(data, dataLength, outSig); + // TODO(yf): remove private key and change to debug + LOG_DEBUG( + GL, + "Signing as replica with " << KVLOG(myId_, seq, rawSigner->getPrivKey(), rawSigner->signatureLength(), result)); return result; } @@ -275,7 +283,7 @@ bool SigManager::verifyReplicaSig(PrincipalId replicaID, size_t dataLength, const concord::Byte* sig, uint16_t sigLength) const { - LOG_INFO(GL, KVLOG(myId_)); + LOG_DEBUG(GL, KVLOG(myId_)); ConcordAssert(replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica()); ConcordAssert(replicasInfo_.isIdOfReplica(replicaID)); int i = 0; @@ -284,15 +292,15 @@ bool SigManager::verifyReplicaSig(PrincipalId replicaID, if (multisigVerifier.get() == nullptr) { continue; } - auto& verifier = reinterpret_cast(multisigVerifier.get())->getVerifier(replicaID); - LOG_INFO(GL, - "Validating as replica with: " << KVLOG( - myId_, replicaID, i, verifier.getPubKey(), sigLength, verifier.signatureLength())); + auto& verifier = multisigVerifier->getVerifier(replicaID); + LOG_DEBUG(GL, + "Validating as replica with: " << KVLOG( + myId_, replicaID, i, verifier.getPubKey(), sigLength, verifier.signatureLength())); if (verifier.verifyBuffer(data, dataLength, sig, sigLength)) { - LOG_INFO(GL, "Validation Successful " << KVLOG(myId_, replicaID, i, sigLength, verifier.signatureLength())); + LOG_DEBUG(GL, "Validation Successful " << KVLOG(myId_, replicaID, i, sigLength, verifier.signatureLength())); return true; } else { - LOG_INFO( + LOG_DEBUG( GL, "Validation failed as replica with: " << KVLOG(myId_, replicaID, i, sigLength, verifier.signatureLength())); } @@ -305,7 +313,7 @@ bool SigManager::verifyReplicaSig(PrincipalId replicaID, // verify using the two last keys, once the last key's checkpoint is reached, the previous key is removed bool SigManager::verifySig( PrincipalId pid, const concord::Byte* data, size_t dataLength, const concord::Byte* sig, uint16_t sigLength) const { - if (replicasInfo_.isIdOfReplica(pid)) { + if (replicasInfo_.isIdOfReplica(pid) && ReplicaConfig::instance().singleSignatureScheme) { return verifyReplicaSig(pid, data, dataLength, sig, sigLength); } @@ -316,23 +324,27 @@ bool SigManager::verifyOwnSignature(const concord::Byte* data, size_t dataLength, const concord::Byte* expectedSignature) const { std::vector sig(getMySigLength()); - for (auto multisigSigner : CryptoManager::instance().getLatestSigners()) { - auto* signer = reinterpret_cast(multisigSigner.get()); - signer->signBuffer(data, dataLength, sig.data()); - - if (std::memcmp(sig.data(), expectedSignature, getMySigLength()) == 0) { - LOG_INFO(GL, "Self-sig validation succeeded"); - return true; - } else { - LOG_INFO(GL, "Self-sig validation failed"); + if (ReplicaConfig::instance().singleSignatureScheme) { + for (auto signer : CryptoManager::instance().getLatestSigners()) { + signer->signBuffer(data, dataLength, sig.data()); + + if (std::memcmp(sig.data(), expectedSignature, getMySigLength()) == 0) { + LOG_DEBUG(GL, "Self-sig validation succeeded"); + return true; + } else { + LOG_DEBUG(GL, "Self-sig validation failed"); + } } + return false; } - return false; + mySigner_->signBuffer(data, dataLength, sig.data()); + return std::memcmp(sig.data(), expectedSignature, getMySigLength()) == 0; } uint16_t SigManager::getMySigLength() const { - if (replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica()) { + if (ReplicaConfig::instance().singleSignatureScheme && + (replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica())) { return getCurrentReplicaSigner()->signatureLength(); } return (uint16_t)mySigner_->signatureLength(); @@ -359,50 +371,51 @@ bool SigManager::hasVerifier(PrincipalId pid) { return verifiers_.find(pid) != v concord::crypto::SignatureAlgorithm SigManager::getMainKeyAlgorithm() const { return concord::crypto::EdDSA; } -std::shared_ptr SigManager::getCurrentReplicaSigner() const { +std::shared_ptr SigManager::getCurrentReplicaSigner() const { + ConcordAssert(ReplicaConfig::instance().singleSignatureScheme); ConcordAssert(replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica()); - std::shared_ptr currentSigner = - std::dynamic_pointer_cast(CryptoManager::instance().getSigner(getReplicaLastExecutedSeq())); - std::shared_ptr ret = currentSigner; - LOG_INFO(GL, KVLOG((uint64_t)ret.get())); - return ret; + auto signer = CryptoManager::instance().getSigner(getReplicaLastExecutedSeq()); + return signer; } -std::shared_ptr SigManager::getLastReplicaSigner() const { +std::shared_ptr SigManager::getLastReplicaSigner() const { + ConcordAssert(ReplicaConfig::instance().singleSignatureScheme); ConcordAssert(replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica()); - std::shared_ptr latestSigner = - std::dynamic_pointer_cast(CryptoManager::instance().getLatestSigners()[0]); - std::shared_ptr ret = latestSigner; - LOG_INFO(GL, KVLOG((uint64_t)ret.get())); - return ret; + auto latestSigner = CryptoManager::instance().getLatestSigners()[0]; + return latestSigner; } std::pair SigManager::getMyLatestPublicKey() const { + ConcordAssert(ReplicaConfig::instance().singleSignatureScheme); ConcordAssert(replicasInfo_.isIdOfReplica(myId_)); auto [seq, latestVerifier] = CryptoManager::instance().getLatestVerifiers()[0]; - auto retVerifier = static_cast(latestVerifier.get()); - LOG_INFO(GL, KVLOG((uint64_t)retVerifier)); - return {seq, retVerifier->getVerifier(myId_).getPubKey()}; + auto pubKey = latestVerifier->getVerifier(myId_).getPubKey(); + LOG_DEBUG(GL, KVLOG(pubKey)); + return {seq, pubKey}; } -std::string SigManager::getSelfPrivKey() const { return getCurrentReplicaSigner()->getPrivKey(); } +std::string SigManager::getSelfPrivKey() const { + if (ReplicaConfig::instance().singleSignatureScheme) { + return getCurrentReplicaSigner()->getPrivKey(); + } + return mySigner_->getPrivKey(); +} std::array SigManager::getPublicKeyOfVerifier(uint32_t id) const { std::array publicKeys; - if (replicasInfo_.isIdOfReplica(id)) { + if (ReplicaConfig::instance().singleSignatureScheme && replicasInfo_.isIdOfReplica(id)) { int i = 0; for (auto [seq, multisigVerifier] : CryptoManager::instance().getLatestVerifiers()) { UNUSED(seq); if (multisigVerifier.get() == nullptr) { continue; } - auto& verifier = reinterpret_cast(multisigVerifier.get())->getVerifier(id); + auto& verifier = multisigVerifier->getVerifier(id); publicKeys[i] = verifier.getPubKey(); ++i; } - } - else if (verifiers_.count(id)) { + } else if (verifiers_.count(id)) { publicKeys[0] = verifiers_.at(id)->getPubKey(); } @@ -414,7 +427,7 @@ const concord::crypto::IVerifier& SigManager::getVerifier(PrincipalId otherPrinc } void SigManager::setReplicaLastExecutedSeq(SeqNum seq) { - ConcordAssert(replicasInfo_.isIdOfReplica(myId_)); + ConcordAssert(replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica()); replicaLastExecutedSeq_ = seq; } diff --git a/bftengine/src/bftengine/SigManager.hpp b/bftengine/src/bftengine/SigManager.hpp index 0787174fbe..96de95ca60 100644 --- a/bftengine/src/bftengine/SigManager.hpp +++ b/bftengine/src/bftengine/SigManager.hpp @@ -29,6 +29,9 @@ using concordMetrics::AtomicCounterHandle; +class EdDSAMultisigSigner; +class EdDSAMultisigVerifier; + namespace bftEngine { namespace impl { @@ -102,8 +105,8 @@ class SigManager { SigManager& operator=(SigManager&&) = delete; concord::crypto::SignatureAlgorithm getMainKeyAlgorithm() const; - std::shared_ptr getCurrentReplicaSigner() const; - std::shared_ptr getLastReplicaSigner() const; + std::shared_ptr getCurrentReplicaSigner() const; + std::shared_ptr getLastReplicaSigner() const; const concord::crypto::IVerifier& getVerifier(PrincipalId otherPrincipal) const; std::string getClientsPublicKeys(); diff --git a/bftengine/src/bftengine/messages/CheckpointMsg.cpp b/bftengine/src/bftengine/messages/CheckpointMsg.cpp index 691783f769..aeb2dd09b2 100644 --- a/bftengine/src/bftengine/messages/CheckpointMsg.cpp +++ b/bftengine/src/bftengine/messages/CheckpointMsg.cpp @@ -79,7 +79,8 @@ void CheckpointMsg::validate(const ReplicasInfo& repInfo) const { void CheckpointMsg::validate(const ReplicasInfo& repInfo, bool validateSignature) const { validateSize(repInfo); - if (validateSignature && !SigManager::instance()->verifySig(idOfGeneratedReplica(), getDataBytes(), getSignatureBytes())) { + if (validateSignature && + !SigManager::instance()->verifySig(idOfGeneratedReplica(), getDataBytes(), getSignatureBytes())) { throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": verifySig(replica)")); } } diff --git a/bftengine/tests/SigManager/SigManager_test.cpp b/bftengine/tests/SigManager/SigManager_test.cpp index e3f4b2f3b3..e69de29bb2 100644 --- a/bftengine/tests/SigManager/SigManager_test.cpp +++ b/bftengine/tests/SigManager/SigManager_test.cpp @@ -1,300 +0,0 @@ -// Concord -// -// Copyright (c) 2021 VMware, Inc. All Rights Reserved. -// -// This product is licensed to you under the Apache 2.0 license (the "License"). You may not use this product except in -// compliance with the Apache 2.0 License. -// -// This product may include a number of subcomponents with separate copyright notices and license terms. Your use of -// these subcomponents is subject to the terms and conditions of the sub-component's license, as noted in the LICENSE -// file. - -#include "SigManager.hpp" -#include "helper.hpp" - -#include - -#include "gtest/gtest.h" -#include "crypto/factory.hpp" -#include "crypto/crypto.hpp" - -using namespace std; -using concord::crypto::KeyFormat; -constexpr size_t RANDOM_DATA_SIZE = 1000U; - -std::default_random_engine generator; - -using concord::crypto::ISigner; -using concord::crypto::IVerifier; -using concord::crypto::Factory; -using bftEngine::ReplicaConfig; -using bftEngine::CryptoManager; -using concord::crypto::SignatureAlgorithm; -using concord::crypto::generateEdDSAKeyPair; - -std::vector> generateKeyPairs(size_t count) { - std::vector> result; - for (size_t i = 0; i < count; i++) { - result.push_back(generateEdDSAKeyPair(KeyFormat::HexaDecimalStrippedFormat)); - } - return result; -} - -void generateRandomData(char* data, size_t len) { - std::uniform_int_distribution distribution(0, 0xFF); - for (size_t i{0}; i < len; ++i) { - data[i] = static_cast(distribution(generator)); - } -} - -void corrupt(concord::Byte* data, size_t len) { - for (size_t i{0}; i < len; ++i) { - data[i] = ~data[i]; - } -} - -void corrupt(char* data, size_t len) { corrupt(reinterpret_cast(data), len); } - -TEST(SignerAndVerifierTest, LoadSignVerifyFromHexKeyPair) { - char data[RANDOM_DATA_SIZE]{0}; - - const auto keyPair = generateEdDSAKeyPair(); - generateRandomData(data, RANDOM_DATA_SIZE); - - const auto signer_ = Factory::getSigner(keyPair.first, ReplicaConfig::instance().replicaMsgSigningAlgo); - const auto verifier_ = Factory::getVerifier(keyPair.second, ReplicaConfig::instance().replicaMsgSigningAlgo); - - // sign with replica signer. - size_t expectedSignerSigLen = signer_->signatureLength(); - std::vector sig(expectedSignerSigLen); - size_t lenRetData; - std::string str_data(data, RANDOM_DATA_SIZE); - lenRetData = signer_->sign(str_data, sig.data()); - ASSERT_EQ(lenRetData, expectedSignerSigLen); - - // validate with replica verifier. - ASSERT_TRUE(verifier_->verify(str_data, sig)); - - // change data randomally, expect failure - char data1[RANDOM_DATA_SIZE]; - std::copy(std::begin(data), std::end(data), std::begin(data1)); - corrupt(data1 + 10, 1); - std::string str_data1(data1, RANDOM_DATA_SIZE); - ASSERT_FALSE(verifier_->verify(str_data1, sig)); - - // change signature randomally, expect failure - corrupt(sig.data(), 1); - str_data = std::string(data, RANDOM_DATA_SIZE); - ASSERT_FALSE(verifier_->verify(str_data, sig)); -} - -TEST(SignerAndVerifierTest, LoadSignVerifyFromPemfiles) { - char data[RANDOM_DATA_SIZE]{0}; - auto keys = generateKeyPairs(1); - auto& [privKey, pubkey] = keys[0]; - - generateRandomData(data, RANDOM_DATA_SIZE); - - const auto signer_ = Factory::getSigner( - privKey, ReplicaConfig::instance().replicaMsgSigningAlgo, KeyFormat::HexaDecimalStrippedFormat); - const auto verifier_ = Factory::getVerifier( - pubkey, ReplicaConfig::instance().replicaMsgSigningAlgo, KeyFormat::HexaDecimalStrippedFormat); - - // sign with replica signer. - size_t expectedSignerSigLen = signer_->signatureLength(); - std::vector sig(expectedSignerSigLen); - std::string str_data(data, RANDOM_DATA_SIZE); - uint32_t lenRetData = signer_->sign(str_data, sig.data()); - ASSERT_EQ(lenRetData, signer_->signatureLength()); - - // validate with replica verifier. - ASSERT_TRUE(verifier_->verify(str_data, sig)); - - // change data randomally, expect failure - char data1[RANDOM_DATA_SIZE]; - std::copy(std::begin(data), std::end(data), std::begin(data1)); - corrupt(data1 + 10, 1); - std::string str_data1(data1, RANDOM_DATA_SIZE); - ASSERT_FALSE(verifier_->verify(str_data1, sig)); - - // change signature randomally, expect failure - corrupt(sig.data(), 1); - str_data = std::string(data, RANDOM_DATA_SIZE); - ASSERT_FALSE(verifier_->verify(str_data, sig)); -} - -class SigManagerTest : public ::testing::Test { - public: - SigManagerTest() : config{createReplicaConfig()}, replicaInfo{config, false, false} {} - - void SetUp() override { - hexKeyPairs = generateKeyPairs(config.numReplicas); - std::set> publicKeysOfReplicas; - std::vector hexReplicaPublicKeys; - - // generateKeyPairs(config.numReplicas + config.numRoReplicas + config.) - for (size_t i = 0; i < config.numReplicas; ++i) { - publicKeysOfReplicas.emplace(i, concord::crypto::EdDSAHexToPem(hexKeyPairs[i]).second); - hexReplicaPublicKeys.push_back(hexKeyPairs[i].second); - } - - sigManager = SigManager::init(config.replicaId, - concord::crypto::EdDSAHexToPem(hexKeyPairs[config.replicaId]).first, - publicKeysOfReplicas, - KeyFormat::PemFormat, - nullptr, - KeyFormat::PemFormat, - replicaInfo); - - bftEngine::CryptoManager::init(std::make_unique( - config.replicaId, hexReplicaPublicKeys, hexKeyPairs[config.replicaId].first)); - } - - protected: - std::vector> hexKeyPairs; - std::shared_ptr sigManager; - const ReplicaConfig& config; - ReplicasInfo replicaInfo; -}; - - -TEST_F(SigManagerTest, TestMySigLength) { - ASSERT_GT(sigManager->getMySigLength(), 0); -} - -TEST_F(SigManagerTest, ReplicasOnlyCheckVerify) { - const size_t numReplicas = config.numReplicas; - std::vector> signers(numReplicas); - - // Load signers to simulate other replicas - for (size_t i = 0; i < numReplicas; ++i) { - signers[i] = Factory::getSigner( - hexKeyPairs[i].first, ReplicaConfig::instance().replicaMsgSigningAlgo, KeyFormat::HexaDecimalStrippedFormat); - } - - std::vector data(RANDOM_DATA_SIZE); - generateRandomData(data.data(), RANDOM_DATA_SIZE); - - for (size_t i = 0; i < numReplicas; ++i) { - const auto& signer = signers[i]; - - // sign with replica signer (other replicas, mock) - std::vector sig(signer->signatureLength()); - ASSERT_EQ(signer->sign(data, sig.data()), sig.size()); - - // Validate with SigManager (my replica) - ASSERT_EQ(sig.size(), sigManager->getSigLength(i)); - ASSERT_TRUE(sigManager->verifySig(i, data, sig)); - - size_t offset = i % data.size(); - corrupt(data.data() + offset, 1); - ASSERT_FALSE(sigManager->verifySig(i, data, sig)); - } -} - -// Check 1 more replica + 1200 clients on 6 additional participants -// where each participant hols a client pool of 200 clients -TEST(SigManagerTestWithClients, ReplicasAndClientsCheckVerify) { - constexpr size_t numReplicas{7}; - constexpr size_t numRoReplicas{2}; - constexpr size_t numOfClientProxies{36}; // (numRoReplicas+numRoReplicas) * 4 - constexpr size_t numParticipantNodes{6}; - constexpr size_t numBftClientsInParticipantNodes{200}; - constexpr size_t totalNumberofExternalBftClients{1200}; // numOfExternaClients * numBftClientsInExternalClient - constexpr PrincipalId myId{0}; - size_t signerIndex{0}; - unique_ptr signers[numReplicas + numParticipantNodes]; // only external clients and consensus replicas sign - - set> publicKeysOfReplicas; - set>> publicKeysOfClients; - unordered_map principalIdToSignerIndex; - - auto keyPairs = generateKeyPairs(numReplicas + numParticipantNodes); - - // Load replica signers to simulate other replicas - PrincipalId currPrincipalId{0}; - for (currPrincipalId = 0; currPrincipalId < numReplicas; ++currPrincipalId) { - signers[signerIndex] = Factory::getSigner(keyPairs[currPrincipalId].first, - ReplicaConfig::instance().replicaMsgSigningAlgo, - KeyFormat::HexaDecimalStrippedFormat); - - publicKeysOfReplicas.insert(make_pair(currPrincipalId, keyPairs[currPrincipalId].second)); - principalIdToSignerIndex.emplace(currPrincipalId, signerIndex); - ++signerIndex; - } - - // Load another group of replica signers to simulate other clients - currPrincipalId = numReplicas + numRoReplicas + numOfClientProxies; - for (size_t i = numReplicas; i < numReplicas + numParticipantNodes; ++i) { - signers[signerIndex] = Factory::getSigner(keyPairs[signerIndex].first, - ReplicaConfig::instance().replicaMsgSigningAlgo, - KeyFormat::HexaDecimalStrippedFormat); - set principalIds; - for (size_t j{0}; j < numBftClientsInParticipantNodes; ++j) { - principalIds.insert(principalIds.end(), currPrincipalId); - principalIdToSignerIndex.insert(make_pair(currPrincipalId, signerIndex)); - ++currPrincipalId; - } - publicKeysOfClients.insert(make_pair(keyPairs[signerIndex].second, std::move(principalIds))); - ++signerIndex; - } - - auto& config = createReplicaConfig(2, 0); - config.numReplicas = numReplicas; - config.numRoReplicas = numRoReplicas; - config.numOfClientProxies = numOfClientProxies; - config.numOfExternalClients = totalNumberofExternalBftClients; - ReplicasInfo replicaInfo(config, false, false); - - shared_ptr sigManager(SigManager::init(myId, - keyPairs[config.replicaId].first, - publicKeysOfReplicas, - KeyFormat::HexaDecimalStrippedFormat, - &publicKeysOfClients, - KeyFormat::HexaDecimalStrippedFormat, - replicaInfo)); - std::vector replicaHexPublicKeys(config.numReplicas); - for (int j = 0; j < config.numReplicas; j++) { - replicaHexPublicKeys[j] = keyPairs[j].second; - } - CryptoManager::init(std::make_unique( - config.replicaId, replicaHexPublicKeys, keyPairs[config.replicaId].second)); - - // principalIdToSignerIndex carries all principal ids for replica, read only replicas and bft-clients. - // There are some principal Ids in the range [minKey(principalIdToSignerIndex), maxKey(principalIdToSignerIndex)] - // which are not recognized by SigManager - these are the principal ids of Proxy Clients. - // In the next loop we will sign 30K times. Every iteration a random principal ID between minKey and maxKey is - // generated. If the ID is valid, we locate the right signer using principalIdToSignerIndex. If not - sign with - // another signer and expect a failure. - PrincipalId minPidInclusive = 0; - PrincipalId maxPidInclusive = currPrincipalId; - std::uniform_int_distribution distribution(minPidInclusive, maxPidInclusive); - std::vector data(RANDOM_DATA_SIZE); - std::vector sig(1024u); - - for (size_t principalId = 0; principalId < maxPidInclusive; ++principalId) { - size_t lenRetData, signerOffset; - - auto iter = principalIdToSignerIndex.find(principalId); - if (iter == principalIdToSignerIndex.end()) { - ASSERT_FALSE(sigManager->hasVerifier(principalId)); - ASSERT_EQ(sigManager->getSigLength(principalId), 0); - continue; - } - signerOffset = iter->second; - - // sign - LOG_DEBUG(GL, KVLOG(principalId, signerOffset, config.replicaId)); - auto expectedSignerSigLen = signers[signerOffset]->signatureLength(); - ASSERT_GE(sig.size(), expectedSignerSigLen); - generateRandomData(data.data(), RANDOM_DATA_SIZE); - lenRetData = signers[signerOffset]->sign(data, sig.data()); - ASSERT_EQ(lenRetData, expectedSignerSigLen); - - // Validate with SigManager (my replica) - ASSERT_TRUE(sigManager->getSigLength(principalId) > 0); - bool signatureValid = sigManager->verifySig( - principalId, data, string_view{reinterpret_cast(sig.data()), sigManager->getSigLength(principalId)}); - ASSERT_TRUE(signatureValid); - } -} diff --git a/bftengine/tests/keyManager/KeyManager_test.cpp b/bftengine/tests/keyManager/KeyManager_test.cpp index f5144cb380..5bd30f2087 100644 --- a/bftengine/tests/keyManager/KeyManager_test.cpp +++ b/bftengine/tests/keyManager/KeyManager_test.cpp @@ -355,7 +355,7 @@ TEST(KeyExchangeManager, initialKeyExchange) { id.sec = std::shared_ptr(new DummyLoaderSaver()); TestKeyManager test{&id}; // get the pub and prv keys from the key handlr and set them to be rotated. - test.km_.sendInitialKey(); + test.km_.waitForQuorumAndTriggerConsensusExchange(); test.km_.futureRet.get(); // set public of replica 0 KeyExchangeMsg kem; @@ -417,7 +417,7 @@ TEST(KeyExchangeManager, endToEnd) { TestKeyManager test{&id}; // set published private key of replica 2 - test.km_.sendInitialKey(); + test.km_.waitForQuorumAndTriggerConsensusExchange(); test.km_.futureRet.get(); // set public of replica 0 diff --git a/client/reconfiguration/src/default_handlers.cpp b/client/reconfiguration/src/default_handlers.cpp index b8b161161f..a33f5b92bf 100644 --- a/client/reconfiguration/src/default_handlers.cpp +++ b/client/reconfiguration/src/default_handlers.cpp @@ -44,7 +44,6 @@ bool ReplicaMainKeyPublicationHandler::validate(const State& state) const { return validateInputState(state); } bool ReplicaMainKeyPublicationHandler::execute(const State& state, WriteState&) { - LOG_INFO(getLogger(), "In ReplicaMainKeyPublicationHandler"); auto cmd = getCmdFromInputState(state); fs::path path = fs::path(output_dir_) / std::to_string(cmd.sender_id); auto curr_key = file_handler_.decryptFile((path / "pub_key").string()).value_or(""); diff --git a/client/reconfiguration/src/poll_based_state_client.cpp b/client/reconfiguration/src/poll_based_state_client.cpp index d0cb6f39de..4b12a93353 100644 --- a/client/reconfiguration/src/poll_based_state_client.cpp +++ b/client/reconfiguration/src/poll_based_state_client.cpp @@ -70,7 +70,7 @@ std::vector PollBasedStateClient::getStateUpdate(bool& succ) const { rreq.sender = id_; rreq.command = creq; auto sn = sn_gen_.unique(); - LOG_INFO(getLogger(), "sending reconfig request" << KVLOG(sn)); + LOG_DEBUG(getLogger(), "sending reconfig request" << KVLOG(sn)); auto rres = sendReconfigurationRequest(rreq, "getStateUpdate-" + std::to_string(sn), sn, true); if (!rres.success) { LOG_WARN(getLogger(), "invalid response from replicas"); @@ -122,12 +122,11 @@ void PollBasedStateClient::start() { } } - LOG_INFO(getLogger(), "Sleeping for" << KVLOG(interval_timeout_ms_)); std::this_thread::sleep_for(std::chrono::milliseconds(interval_timeout_ms_)); if (stopped) break; std::lock_guard lk(lock_); bool succ; - LOG_INFO(getLogger(), "Getting state update"); + LOG_DEBUG(getLogger(), "Getting state update"); auto new_state = getStateUpdate(succ); uint64_t max_update_block{0}; for (const auto& s : new_state) { diff --git a/kvbc/include/categorization/blockchain.h b/kvbc/include/categorization/blockchain.h index 5f45cfb9c6..4af3bef18d 100644 --- a/kvbc/include/categorization/blockchain.h +++ b/kvbc/include/categorization/blockchain.h @@ -49,8 +49,7 @@ class Blockchain { } void addBlock(const Block& new_block, storage::rocksdb::NativeWriteBatch& wb) { - LOG_INFO(GL, "Adding block_id: " << new_block.id() << " to the database"); - printCallStack(); + LOG_DEBUG(GL, "Adding block_id: " << new_block.id() << " to the database"); wb.put(detail::BLOCKS_CF, Block::generateKey(new_block.id()), Block::serialize(new_block)); } diff --git a/kvbc/src/reconfiguration_kvbc_handler.cpp b/kvbc/src/reconfiguration_kvbc_handler.cpp index 8606fe0f68..d08a05bacc 100644 --- a/kvbc/src/reconfiguration_kvbc_handler.cpp +++ b/kvbc/src/reconfiguration_kvbc_handler.cpp @@ -423,8 +423,7 @@ concord::messages::ClientStateReply KvbcClientReconfigurationHandler::buildRepli creply.block_id = 0; auto res = ro_storage_.getLatest(concord::kvbc::categorization::kConcordReconfigurationCategoryId, command_type + std::to_string(clientid)); - LOG_INFO(GL, "Building state reply" << KVLOG(clientid, command_type, res.has_value())); - printCallStack(); + LOG_DEBUG(GL, "Building state reply" << KVLOG(clientid, command_type, res.has_value())); if (res.has_value()) { std::visit( [&](auto&& arg) { @@ -468,7 +467,7 @@ bool KvbcClientReconfigurationHandler::handle(const concord::messages::ClientRec concord::messages::ReconfigurationResponse& rres) { concord::messages::ClientReconfigurationStateReply rep; uint16_t first_client_id = ReplicaConfig::instance().numReplicas + ReplicaConfig::instance().numRoReplicas; - LOG_INFO(GL, KVLOG(first_client_id, sender_id)); + LOG_DEBUG(GL, "Handling ClientReconfigurationStateRequest" << KVLOG(first_client_id, sender_id)); if (sender_id > first_client_id) { for (uint8_t i = kvbc::keyTypes::CLIENT_COMMAND_TYPES::start_ + 1; i < kvbc::keyTypes::CLIENT_COMMAND_TYPES::end_; i++) { @@ -499,7 +498,7 @@ bool KvbcClientReconfigurationHandler::handle(const concord::messages::ClientRec for (auto& update_type : non_external_client_update_types) { auto state_reply = buildReplicaStateReply(update_type, i); if (state_reply.block_id > 0) { - LOG_INFO(GL, "Got State reply for replica id" << i); + LOG_DEBUG(GL, "Got State reply for replica id" << i); rep.states.push_back(state_reply); } } diff --git a/kvbc/src/v4blockchain/v4_blockchain.cpp b/kvbc/src/v4blockchain/v4_blockchain.cpp index 0fb9012b17..37937d731c 100644 --- a/kvbc/src/v4blockchain/v4_blockchain.cpp +++ b/kvbc/src/v4blockchain/v4_blockchain.cpp @@ -99,8 +99,7 @@ BlockId KeyValueBlockchain::add(categorization::Updates &&updates) { if (block_id % 100 == 0) { v4_metrics_comp_.UpdateAggregator(); } - LOG_INFO(GL, "Adding block to DB with" << KVLOG(sequence_number, block_id)); - printCallStack(); + LOG_DEBUG(GL, "Adding block to DB with" << KVLOG(sequence_number, block_id)); return block_id; } diff --git a/libs/communication/StateControl.hpp b/libs/communication/StateControl.hpp index 2595fe9c82..b659db530a 100644 --- a/libs/communication/StateControl.hpp +++ b/libs/communication/StateControl.hpp @@ -73,7 +73,9 @@ class StateControl { } return str; } - void setGetPeerPubKeyMethod(std::function(uint32_t)> m) { get_peer_pub_key_ = std::move(m); } + void setGetPeerPubKeyMethod(std::function(uint32_t)> m) { + get_peer_pub_key_ = std::move(m); + } std::array getPeerPubKey(uint32_t id) { if (get_peer_pub_key_) return get_peer_pub_key_(id); diff --git a/libs/crypto/openssl/EdDSASigner.hpp b/libs/crypto/openssl/EdDSASigner.hpp index 97c9e84eaf..fbd9345823 100644 --- a/libs/crypto/openssl/EdDSASigner.hpp +++ b/libs/crypto/openssl/EdDSASigner.hpp @@ -36,7 +36,8 @@ class EdDSASigner : public ISigner { explicit EdDSASigner(const PrivateKeyType &privateKey) : privateKey_(privateKey) {} size_t signBuffer(const concord::Byte *msg, size_t len, concord::Byte *signature) const override { - LOG_INFO(GL, "Signing with key " << getPrivKey()); + // TODO: remove + LOG_DEBUG(GL, "Signing with key " << getPrivKey()); UniquePKEY pkey(EVP_PKEY_new_raw_private_key( NID_ED25519, nullptr, privateKey_.getBytes().data(), privateKey_.getBytes().size())); ConcordAssertNE(pkey, nullptr); diff --git a/libs/crypto/openssl/EdDSAVerifier.hpp b/libs/crypto/openssl/EdDSAVerifier.hpp index d5934b3d28..914639e055 100644 --- a/libs/crypto/openssl/EdDSAVerifier.hpp +++ b/libs/crypto/openssl/EdDSAVerifier.hpp @@ -39,7 +39,7 @@ class EdDSAVerifier : public IVerifier { UniqueContext ctx{EVP_MD_CTX_new()}; ConcordAssertEQ(EVP_DigestVerifyInit(ctx.get(), nullptr, nullptr, nullptr, pkey.get()), OPENSSL_SUCCESS); auto result = OPENSSL_SUCCESS == EVP_DigestVerify(ctx.get(), sig, sigLen, msg, msgLen); - LOG_INFO(GL, "Verification complete" << KVLOG(getPubKey(), result)); + LOG_DEBUG(GL, "Verification complete" << KVLOG(getPubKey(), result)); return result; } diff --git a/libs/crypto/src/threshsign/ThresholdSignaturesTypes.cpp b/libs/crypto/src/threshsign/ThresholdSignaturesTypes.cpp index 4374934b6f..57c64abe35 100644 --- a/libs/crypto/src/threshsign/ThresholdSignaturesTypes.cpp +++ b/libs/crypto/src/threshsign/ThresholdSignaturesTypes.cpp @@ -107,6 +107,8 @@ std::vector Cryptosystem::getSystemVerificationKeys() const { return verificationKeys_; } +std::string Cryptosystem::getMyVerificationKey() const { return getSystemVerificationKeys()[signerID_]; } + std::vector Cryptosystem::getSystemPrivateKeys() const { std::vector output; if (privateKeys_.size() != static_cast(numSigners_)) { @@ -120,7 +122,7 @@ std::vector Cryptosystem::getSystemPrivateKeys() const { } std::string Cryptosystem::getPrivateKey(uint16_t signerIndex) const { - if ((signerIndex < 0) || (signerIndex >= numSigners_)) + if (signerIndex >= numSigners_) throw std::out_of_range(__PRETTY_FUNCTION__ + std::string(" Signer index out of range: ") + std::to_string(signerIndex)); diff --git a/libs/crypto/src/threshsign/eddsa/EdDSAMultisigSigner.cpp b/libs/crypto/src/threshsign/eddsa/EdDSAMultisigSigner.cpp index 43dbf4f10c..3f4eedfe5e 100644 --- a/libs/crypto/src/threshsign/eddsa/EdDSAMultisigSigner.cpp +++ b/libs/crypto/src/threshsign/eddsa/EdDSAMultisigSigner.cpp @@ -17,8 +17,8 @@ using concord::crypto::openssl::EdDSASigner; EdDSAMultisigSigner::EdDSAMultisigSigner(const EdDSAThreshsignPrivateKey &privateKey, const uint32_t id) : EdDSASigner{privateKey}, publicKey_{}, id_{id} { - LOG_INFO(EDDSA_MULTISIG_LOG, "created eddsa signer with " << KVLOG(id_, getPrivKey())); - printCallStack(); + // TODO: remove log message + LOG_DEBUG(EDDSA_MULTISIG_LOG, "created eddsa signer with " << KVLOG(id_, getPrivKey())); } int EdDSAMultisigSigner::requiredLengthForSignedData() const { return sizeof(SingleEdDSASignature); } diff --git a/libs/crypto/src/threshsign/eddsa/EdDSAMultisigVerifier.cpp b/libs/crypto/src/threshsign/eddsa/EdDSAMultisigVerifier.cpp index 01979997d4..d16093f372 100644 --- a/libs/crypto/src/threshsign/eddsa/EdDSAMultisigVerifier.cpp +++ b/libs/crypto/src/threshsign/eddsa/EdDSAMultisigVerifier.cpp @@ -27,13 +27,13 @@ int EdDSASignatureAccumulator::add(const char *sigShareWithId, int len) { reinterpret_cast(expectedMsgDigest_.data()), expectedMsgDigest_.size(), singleSignature); if (!result) { invalidShares_.insert(static_cast(singleSignature.id)); - LOG_INFO(EDDSA_MULTISIG_LOG, "Share id: " << singleSignature.id << " is invalid"); + LOG_DEBUG(EDDSA_MULTISIG_LOG, "Share id: " << singleSignature.id << " is invalid"); } } auto result = signatures_.insert({singleSignature.id, singleSignature}); if (result.second) { - LOG_INFO(EDDSA_MULTISIG_LOG, "Added " << KVLOG(this, singleSignature.id, signatures_.size())); + LOG_DEBUG(EDDSA_MULTISIG_LOG, "Added " << KVLOG(this, singleSignature.id, signatures_.size())); } return static_cast(signatures_.size()); } @@ -43,7 +43,7 @@ void EdDSASignatureAccumulator::setExpectedDigest(const unsigned char *msg, int } size_t EdDSASignatureAccumulator::getFullSignedData(char *outThreshSig, int threshSigLen) { - LOG_INFO(EDDSA_MULTISIG_LOG, KVLOG(threshSigLen, signatures_.size())); + LOG_DEBUG(EDDSA_MULTISIG_LOG, KVLOG(threshSigLen, signatures_.size())); ConcordAssertGE(static_cast(threshSigLen), static_cast(signatures_.size()) * sizeof(SingleEdDSASignature)); size_t offset = 0; @@ -109,13 +109,13 @@ bool EdDSAMultisigVerifier::verify(const char *msg, int msgLen, const char *sig, continue; } auto result = verifySingleSignature(reinterpret_cast(msg), msgLenUnsigned, currentSignature); - LOG_INFO(EDDSA_MULTISIG_LOG, "Verified id: " << KVLOG(currentSignature.id, result)); + LOG_DEBUG(EDDSA_MULTISIG_LOG, "Verified id: " << KVLOG(currentSignature.id, result, signatureCountInBuffer)); validSignatureCount += result == true; } bool result = validSignatureCount >= threshold_; - LOG_INFO(EDDSA_MULTISIG_LOG, KVLOG(validSignatureCount, threshold_)); + LOG_DEBUG(EDDSA_MULTISIG_LOG, KVLOG(validSignatureCount, threshold_, signatureCountInBuffer, signersCount_)); return result; } diff --git a/libs/crypto/threshsign/ThresholdSignaturesTypes.h b/libs/crypto/threshsign/ThresholdSignaturesTypes.h index 2bcfb3f7b7..3b3c611d1a 100644 --- a/libs/crypto/threshsign/ThresholdSignaturesTypes.h +++ b/libs/crypto/threshsign/ThresholdSignaturesTypes.h @@ -179,9 +179,7 @@ class Cryptosystem { * represented as strings. Their format is cryptosystem type-dependent. * * @return A vector containing the verification keys, in order of which signer - * they correspond to. To comply with the convention of 1-indexing - * signer IDs, verification keys will begin at index 1 of the vector. - * The contents of index 0 of the vector is left undefined. + * they correspond to. * * @throws std::runtime_error If this cryptosystem does not * currently have verification @@ -191,6 +189,12 @@ class Cryptosystem { */ std::vector getSystemVerificationKeys() const; + /** + * + * @return The verification key of the current replica + */ + std::string getMyVerificationKey() const; + /** * Get a list of private keys for this threshold cryptosystem, represented as * strings. Their format is cryptosystem type-dependent. diff --git a/libs/util/assertUtils.hpp b/libs/util/assertUtils.hpp index 43e4682165..b42ba0cfa8 100644 --- a/libs/util/assertUtils.hpp +++ b/libs/util/assertUtils.hpp @@ -23,51 +23,6 @@ #include "util/kvstream.h" inline void printCallStack() { - /*const uint32_t MAX_FRAMES = 100; - void *addrlist[MAX_FRAMES]; - int addrLen = backtrace(addrlist, MAX_FRAMES); - if (addrLen) { - char **symbolsList = backtrace_symbols(addrlist, addrLen); - if (symbolsList) { - std::ostringstream os; - const size_t MAX_FUNC_NAME_SIZE = 256; - // Iterate over the returned symbol lines. Skip the first, it is the address of this function. - for (int i = 1; i < addrLen; i++) { - char *beginName = nullptr, *beginOffset = nullptr, *endOffset = nullptr; - for (char *ptr = symbolsList[i]; *ptr; ++ptr) { - if (*ptr == '(') - beginName = ptr; - else if (*ptr == '+') - beginOffset = ptr; - else if (*ptr == ')' && beginOffset) { - endOffset = ptr; - break; - } - } - if (beginName && beginOffset && endOffset && beginName < beginOffset) { - *beginName++ = '\0'; - *beginOffset++ = '\0'; - *endOffset = '\0'; - int status; - size_t demangledSize; - char *ret = abi::__cxa_demangle(beginName, nullptr, &demangledSize, &status); - if (status == 0) { - if (demangledSize > MAX_FUNC_NAME_SIZE) { - ret[MAX_FUNC_NAME_SIZE] = '\0'; - } - os << " [bt] " << ret << "+" << beginOffset << std::endl; - } - free(ret); - } - } - LOG_INFO(GL, "\n" << os.str()); - std::free(symbolsList); - } - }*/ - return; -} - -inline void printCallStack2() { const uint32_t MAX_FRAMES = 100; void *addrlist[MAX_FRAMES]; int addrLen = backtrace(addrlist, MAX_FRAMES); diff --git a/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp b/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp index 88c4ec04dc..3629e9d291 100644 --- a/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp +++ b/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp @@ -273,6 +273,7 @@ std::optional> InternalCommandsHandler::getBl if (concordUpdates) { const auto &u = std::get(concordUpdates->get()); for (const auto &[key, valueWithFlags] : u.kv) { + UNUSED(valueWithFlags); if (key[0] == concord::kvbc::keyTypes::reconfiguration_rep_main_key) { // Test categories and deployment categories are expected to be mutually exclusive ConcordAssert(ret.empty()); @@ -283,7 +284,6 @@ std::optional> InternalCommandsHandler::getBl } } - return ret; } @@ -479,8 +479,7 @@ OperationResult InternalCommandsHandler::executeWriteCommand(uint32_t requestSiz // The batched requests all share batchCid // Batched requests are handled by the preprocessor and should not be sent more than once hasConflict = hasConflict || (!isFirstClientRequest && m_clientToMaxExecutedReqId[clientId] > batchCid); - } - else { + } else { hasConflict = hasConflict || (!isFirstClientRequest && m_clientToMaxExecutedReqId[clientId] >= requestId); } @@ -504,6 +503,7 @@ OperationResult InternalCommandsHandler::executeWriteCommand(uint32_t requestSiz if (!hasConflict) { write_rep.latest_block = currBlock + 1; auto [iter, isNew] = m_clientToMaxExecutedReqId.emplace(clientId, 0); + UNUSED(iter); UNUSED(isNew); m_clientToMaxExecutedReqId[clientId] = std::max(m_clientToMaxExecutedReqId[clientId], batchCid); } else { diff --git a/tests/simpleKVBC/TesterReplica/internalCommandsHandler.hpp b/tests/simpleKVBC/TesterReplica/internalCommandsHandler.hpp index 5e3010a514..99f6ab2527 100644 --- a/tests/simpleKVBC/TesterReplica/internalCommandsHandler.hpp +++ b/tests/simpleKVBC/TesterReplica/internalCommandsHandler.hpp @@ -173,5 +173,5 @@ class InternalCommandsHandler : public concord::kvbc::ICommandsHandler { // Some tests expect every block to be created by a request issued by test clients. // However, internal communication between replicas can also create blocks, e.g: // When rotating keys. - static constexpr const char* s_ignoreBlockStr = "ignoreBlock"; + static constexpr const char *s_ignoreBlockStr = "ignoreBlock"; }; diff --git a/tests/simpleKVBC/TesterReplica/main.cpp b/tests/simpleKVBC/TesterReplica/main.cpp index f2785e9f63..c91a8df856 100644 --- a/tests/simpleKVBC/TesterReplica/main.cpp +++ b/tests/simpleKVBC/TesterReplica/main.cpp @@ -194,7 +194,20 @@ static void signal_handler(int signal_num) { } // namespace int main(int argc, char** argv) { - const auto signals = {SIGABRT, SIGILL, SIGFPE, SIGSEGV, SIGTERM, SIGKILL, SIGQUIT, SIGHUP, SIGBUS, SIGSYS, SIGPIPE, SIGSTOP, SIGTSTP, SIGXFSZ}; + const auto signals = {SIGABRT, + SIGILL, + SIGFPE, + SIGSEGV, + SIGTERM, + SIGKILL, + SIGQUIT, + SIGHUP, + SIGBUS, + SIGSYS, + SIGPIPE, + SIGSTOP, + SIGTSTP, + SIGXFSZ}; for (int signalCode : signals) { defaultHandlers[signalCode] = signal(signalCode, signal_handler); From cc5a2f351222055372536df6194dbea3af26718e Mon Sep 17 00:00:00 2001 From: Ynon Flum Date: Sun, 19 Feb 2023 17:50:13 +0200 Subject: [PATCH 05/18] Make replicas wait for quorum before publishing main key when starting Move message validation to ReplicaBase Update RO replicas key state after ST via cre (same mechanism as clients) --- .../include/bftengine/KeyExchangeManager.hpp | 5 +++- .../bcstatetransfer/AsyncStateTransferCRE.cpp | 8 +++++-- .../src/bftengine/KeyExchangeManager.cpp | 14 +++++++---- bftengine/src/bftengine/ReadOnlyReplica.cpp | 6 ----- bftengine/src/bftengine/ReplicaBase.hpp | 23 +++++++++++++++++-- .../src/bftengine/ReplicaForStateTransfer.cpp | 13 ++++------- bftengine/src/bftengine/ReplicaImp.cpp | 19 +-------------- bftengine/src/bftengine/RequestHandler.cpp | 2 +- bftengine/src/bftengine/SigManager.cpp | 3 ++- .../preprocessor/tests/preprocessor_test.cpp | 1 - kvbc/src/reconfiguration_kvbc_handler.cpp | 1 + .../integrity_checker.cpp | 1 - tests/apollo/util/bft.py | 1 - tests/simpleKVBC/TesterReplica/main.cpp | 9 ++++---- 14 files changed, 56 insertions(+), 50 deletions(-) diff --git a/bftengine/include/bftengine/KeyExchangeManager.hpp b/bftengine/include/bftengine/KeyExchangeManager.hpp index 7d54617856..f374e163aa 100644 --- a/bftengine/include/bftengine/KeyExchangeManager.hpp +++ b/bftengine/include/bftengine/KeyExchangeManager.hpp @@ -35,8 +35,11 @@ class KeyExchangeManager { void generateConsensusKeyAndSendInternalClientMsg(const SeqNum& sn); // Send the current main public key of the replica to consensus void sendMainPublicKey(); + + void waitForQuorum(const ReplicaImp* repImpInstance); + // Waits for a quorum and calls generateConsensusKeyAndSendInternalClientMsg - void waitForQuorumAndTriggerConsensusExchange(const ReplicaImp* repImpInstance, const SeqNum& = 0); + void waitForQuorumAndTriggerConsensusExchange(const ReplicaImp* repImpInstance, const SeqNum s = 0); // The execution handler implementation that is called after a key exchange msg passed consensus. // The new key pair will be used from two checkpoints after kemsg.generated_sn std::string onKeyExchange(const KeyExchangeMsg& kemsg, const SeqNum& req_sn, const std::string& cid); diff --git a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp index ac5076b34a..3afed4d5ab 100644 --- a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp +++ b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp @@ -130,6 +130,7 @@ class ScalingReplicaHandler : public IStateHandler { } }; +// TODO(yf): remove class MainKeyUpdateHandler : public IStateHandler { public: MainKeyUpdateHandler() { LOG_INFO(getLogger(), "Created StateTransfer CRE replica main key update handler"); } @@ -186,8 +187,11 @@ std::shared_ptr CreFactory::create( IStateClient* pbc = new PollBasedStateClient(bftClient, cre_config.interval_timeout_ms_, 0, cre_config.id_); auto cre = std::make_shared(cre_config, pbc, std::make_shared()); - if (!bftEngine::ReplicaConfig::instance().isReadOnly) cre->registerHandler(std::make_shared()); - cre->registerHandler(std::make_shared()); + if (bftEngine::ReplicaConfig::instance().isReadOnly) { + cre->registerHandler(std::make_shared()); + } else { + cre->registerHandler(std::make_shared()); + } return cre; } diff --git a/bftengine/src/bftengine/KeyExchangeManager.cpp b/bftengine/src/bftengine/KeyExchangeManager.cpp index a91b996de3..77bca63157 100644 --- a/bftengine/src/bftengine/KeyExchangeManager.cpp +++ b/bftengine/src/bftengine/KeyExchangeManager.cpp @@ -355,14 +355,20 @@ void KeyExchangeManager::loadClientPublicKey(const std::string& key, if (saveToReservedPages) saveClientsPublicKeys(SigManager::instance()->getClientsPublicKeys()); } -void KeyExchangeManager::waitForQuorumAndTriggerConsensusExchange(const ReplicaImp* repImpInstance, const SeqNum& s) { - std::unique_lock lock(startup_mutex_); - SCOPED_MDC(MDC_REPLICA_ID_KEY, std::to_string(ReplicaConfig::instance().replicaId)); - if (!ReplicaConfig::instance().waitForFullCommOnStartup) { +void KeyExchangeManager::waitForQuorum(const ReplicaImp* repImpInstance) { + bool partialQuorum = !ReplicaConfig::instance().waitForFullCommOnStartup; + LOG_INFO(KEY_EX_LOG, "Waiting for quorum" << KVLOG(partialQuorum)); + if (partialQuorum) { waitForLiveQuorum(repImpInstance); } else { waitForFullCommunication(); } +} + +void KeyExchangeManager::waitForQuorumAndTriggerConsensusExchange(const ReplicaImp* repImpInstance, const SeqNum s) { + std::unique_lock lock(startup_mutex_); + SCOPED_MDC(MDC_REPLICA_ID_KEY, std::to_string(ReplicaConfig::instance().replicaId)); + waitForQuorum(repImpInstance); generateConsensusKeyAndSendInternalClientMsg(s); metrics_->sent_key_exchange_on_start_status.Get().Set("True"); diff --git a/bftengine/src/bftengine/ReadOnlyReplica.cpp b/bftengine/src/bftengine/ReadOnlyReplica.cpp index 1e36e441c5..82d234163f 100644 --- a/bftengine/src/bftengine/ReadOnlyReplica.cpp +++ b/bftengine/src/bftengine/ReadOnlyReplica.cpp @@ -60,12 +60,6 @@ ReadOnlyReplica::ReadOnlyReplica(const ReplicaConfig &config, msgHandlers_->registerMsgHandler( MsgCode::StateTransfer, std::bind(&ReadOnlyReplica::messageHandler, this, std::placeholders::_1)); - msgHandlers_->registerMsgHandler( - MsgCode::StateTransfer, - std::bind(&ReadOnlyReplica::messageHandler, this, std::placeholders::_1)); - msgHandlers_->registerMsgHandler( - MsgCode::StateTransfer, - std::bind(&ReadOnlyReplica::messageHandler, this, std::placeholders::_1)); metrics_.Register(); SigManager::init(config_.replicaId, diff --git a/bftengine/src/bftengine/ReplicaBase.hpp b/bftengine/src/bftengine/ReplicaBase.hpp index 7799fb4820..86cd8d0f81 100644 --- a/bftengine/src/bftengine/ReplicaBase.hpp +++ b/bftengine/src/bftengine/ReplicaBase.hpp @@ -26,6 +26,7 @@ namespace bftEngine::impl { class MsgHandlersRegistrator; class MsgsCommunicator; class ReplicasInfo; +class CheckpointMsg; using concordMetrics::GaugeHandle; using concordMetrics::StatusHandle; @@ -80,7 +81,25 @@ class ReplicaBase { void sendRaw(MessageBase* m, NodeIdType dest); - bool validateMessage(MessageBase* msg) { + template + bool validateMessage(MessageType* msg) { + if (config_.debugStatisticsEnabled) { + DebugStatistics::onReceivedExMessage(msg->type()); + } + try { + if constexpr (std::is_same_v) { + msg->validate(*repsInfo, false); + } else { + msg->validate(*repsInfo); + } + return true; + } catch (std::exception& e) { + onReportAboutInvalidMessage(msg, e.what()); + return false; + } + } + + /*bool validateMessage(MessageBase* msg) { try { if (config_.debugStatisticsEnabled) DebugStatistics::onReceivedExMessage(msg->type()); @@ -90,7 +109,7 @@ class ReplicaBase { onReportAboutInvalidMessage(msg, e.what()); return false; } - } + }*/ protected: static const uint16_t ALL_OTHER_REPLICAS = UINT16_MAX; diff --git a/bftengine/src/bftengine/ReplicaForStateTransfer.cpp b/bftengine/src/bftengine/ReplicaForStateTransfer.cpp index 7fe7ee2767..dea76c0e72 100644 --- a/bftengine/src/bftengine/ReplicaForStateTransfer.cpp +++ b/bftengine/src/bftengine/ReplicaForStateTransfer.cpp @@ -87,23 +87,20 @@ void ReplicaForStateTransfer::start() { if (!config_.isReadOnly) { // Load the public keys of the other replicas from reserved pages // so that their responses can be validated - cre_->halt(); KeyExchangeManager::instance().loadPublicKeys(); + + // Make sure to sign the reconfiguration client messages using the key + // other replicas expect + SigManager::instance()->setReplicaLastExecutedSeq(checkpoint * checkpointWindowSize); + // Need to update private key to match the loaded public key in case they differ (key exchange was executed // on other replicas but not on this one, finishing ST does not mean that missed key exchanges are executed) // This can be done by iterating the saved cryptosystems and updating their private key if their // public key matches the candidate saved in KeyExchangeManager - - // Clear old keys CryptoManager::instance().onCheckpoint(checkpoint); auto [priv, pub] = KeyExchangeManager::instance().getCandidateKeyPair(); CryptoManager::instance().syncPrivateKeyAfterST(priv, pub); - // Make sure to sign the reconfiguration client messages using the key - // other replicas expect - SigManager::instance()->setReplicaLastExecutedSeq(checkpoint * checkpointWindowSize); - cre_->resume(); - // At this point, we, if are not going to have another blocks in state transfer. So, we can safely stop CRE. // if there is a reconfiguration state change that prevents us from starting another state transfer (i.e. // scaling) then CRE probably won't work as well. diff --git a/bftengine/src/bftengine/ReplicaImp.cpp b/bftengine/src/bftengine/ReplicaImp.cpp index 5d130e8f87..f2be781467 100644 --- a/bftengine/src/bftengine/ReplicaImp.cpp +++ b/bftengine/src/bftengine/ReplicaImp.cpp @@ -260,24 +260,6 @@ void ReplicaImp::validatedMessageHandler(CarrierMesssage *msg) { } } -template -bool ReplicaImp::validateMessage(MessageType *msg) { - if (config_.debugStatisticsEnabled) { - DebugStatistics::onReceivedExMessage(msg->type()); - } - try { - if constexpr (std::is_same_v) { - msg->validate(*repsInfo, false); - } else { - msg->validate(*repsInfo); - } - return true; - } catch (std::exception &e) { - onReportAboutInvalidMessage(msg, e.what()); - return false; - } -} - /** * asyncValidateMessage This is a family of asynchronous message which just schedules * the validation in a thread bag and returns peacefully. This will also translate the message @@ -4665,6 +4647,7 @@ void ReplicaImp::start() { // If key exchange is disabled, first publish the replica's main key to clients if (ReplicaConfig::instance().singleSignatureScheme || ReplicaConfig::instance().publishReplicasMasterKeyOnStartup) { + KeyExchangeManager::instance().waitForQuorum(this); KeyExchangeManager::instance().sendMainPublicKey(); } } diff --git a/bftengine/src/bftengine/RequestHandler.cpp b/bftengine/src/bftengine/RequestHandler.cpp index c4d759075f..3e0da5173b 100644 --- a/bftengine/src/bftengine/RequestHandler.cpp +++ b/bftengine/src/bftengine/RequestHandler.cpp @@ -161,7 +161,7 @@ void RequestHandler::execute(IRequestsHandler::ExecutionRequestsQueue& requests, } else { // this replica has not reached stable seqNum yet to create snapshot at requested seqNum // add a callback to be called when seqNum is stable. We need to create snapshot on stable - // seq num because checkpoint msg certificate is stored on stable seq num and is used for intergrity + // seq num because checkpoint msg certificate is stored on stable seq num and is used for integrity // check of db snapshots const auto& seqNumToCreateSanpshot = createDbChkPtMsg.seqNum; DbCheckpointManager::instance().setCheckpointInProcess(true, *blockId); diff --git a/bftengine/src/bftengine/SigManager.cpp b/bftengine/src/bftengine/SigManager.cpp index 2860008199..955e3d7444 100644 --- a/bftengine/src/bftengine/SigManager.cpp +++ b/bftengine/src/bftengine/SigManager.cpp @@ -325,7 +325,8 @@ bool SigManager::verifyOwnSignature(const concord::Byte* data, const concord::Byte* expectedSignature) const { std::vector sig(getMySigLength()); if (ReplicaConfig::instance().singleSignatureScheme) { - for (auto signer : CryptoManager::instance().getLatestSigners()) { + auto signers = CryptoManager::instance().getLatestSigners(); + for (auto& signer : signers) { signer->signBuffer(data, dataLength, sig.data()); if (std::memcmp(sig.data(), expectedSignature, getMySigLength()) == 0) { diff --git a/bftengine/src/preprocessor/tests/preprocessor_test.cpp b/bftengine/src/preprocessor/tests/preprocessor_test.cpp index 99194f80c3..0ede21dddd 100644 --- a/bftengine/src/preprocessor/tests/preprocessor_test.cpp +++ b/bftengine/src/preprocessor/tests/preprocessor_test.cpp @@ -36,7 +36,6 @@ using namespace std; using namespace bft::communication; using namespace bftEngine; using namespace preprocessor; -using concord::crypto::SignatureAlgorithm; namespace { diff --git a/kvbc/src/reconfiguration_kvbc_handler.cpp b/kvbc/src/reconfiguration_kvbc_handler.cpp index d08a05bacc..71c688c8af 100644 --- a/kvbc/src/reconfiguration_kvbc_handler.cpp +++ b/kvbc/src/reconfiguration_kvbc_handler.cpp @@ -443,6 +443,7 @@ concord::messages::ClientStateReply KvbcClientReconfigurationHandler::buildRepli } else if (command_type == std::string{kvbc::keyTypes::reconfiguration_rep_main_key}) { concord::messages::ReplicaMainKeyUpdate cmd; concord::messages::deserialize(data_buf, cmd); + cmd.seq_num -= (cmd.seq_num < 2 * checkpointWindowSize ? 0 : 2 * checkpointWindowSize); creply.response = cmd; } auto epoch_data = diff --git a/kvbc/tools/object_store_utility/integrity_checker.cpp b/kvbc/tools/object_store_utility/integrity_checker.cpp index 78d7f55bdf..1d4b7777ba 100644 --- a/kvbc/tools/object_store_utility/integrity_checker.cpp +++ b/kvbc/tools/object_store_utility/integrity_checker.cpp @@ -28,7 +28,6 @@ using namespace std::placeholders; using concordUtils::Status; using bftEngine::bcst::impl::BCStateTran; using kvbc::v1DirectKeyValue::S3StorageFactory; -using crypto::KeyFormat; void IntegrityChecker::initKeysConfig(const fs::path& keys_file) { LOG_DEBUG(logger_, keys_file); diff --git a/tests/apollo/util/bft.py b/tests/apollo/util/bft.py index d1e6981c02..b102332e1a 100644 --- a/tests/apollo/util/bft.py +++ b/tests/apollo/util/bft.py @@ -519,7 +519,6 @@ async def change_configuration(self, config, generate_tls=False, use_unified_cer # Generate certificates for replicas, clients, and reserved clients self.generate_tls_certs(self.num_total_replicas() + config.num_clients + RESERVED_CLIENTS_QUOTA + generate_cre, use_unified_certs=use_unified_certs) - @log_call def restart_clients(self, generate_tx_signing_keys=True, restart_replicas=True): with log.start_action(action_type="restart_clients", generate_tx_signing_keys=generate_tx_signing_keys, restart_replicas=restart_replicas): diff --git a/tests/simpleKVBC/TesterReplica/main.cpp b/tests/simpleKVBC/TesterReplica/main.cpp index c91a8df856..2b1de7a215 100644 --- a/tests/simpleKVBC/TesterReplica/main.cpp +++ b/tests/simpleKVBC/TesterReplica/main.cpp @@ -118,6 +118,10 @@ void run_replica(int argc, char** argv) { MDC_PUT(MDC_REPLICA_ID_KEY, std::to_string(setup->GetReplicaConfig().replicaId)); MDC_PUT(MDC_THREAD_KEY, "main"); + // Start metrics server before the creation of the replica so that we handle the case where + // a replica waits for an unresponsive primary on startup but still updates state-transfer related metrics + setup->GetMetricsServer().Start(); + replica = std::make_shared( setup->GetCommunication(), setup->GetReplicaConfig(), @@ -167,10 +171,7 @@ void run_replica(int argc, char** argv) { // Setup a test cron table, if requested in configuration. cronSetup(*setup, *replica); - // Start metrics server after creation of the replica so that we ensure - // registration of metrics from the replica with the aggregator and don't - // return empty metrics from the metrics server. - setup->GetMetricsServer().Start(); + while (replica->isRunning()) { if (timeToExit) { setup->GetMetricsServer().Stop(); From 50fcb1784281bb238838801f19110fc5c5aeff58 Mon Sep 17 00:00:00 2001 From: Ynon Flum Date: Wed, 22 Feb 2023 14:51:05 +0200 Subject: [PATCH 06/18] Fixed client message replay execution after state transfer --- bftengine/src/bftengine/ReplicaImp.cpp | 19 +++-- bftengine/src/bftengine/ReplicaImp.hpp | 1 + .../TesterReplica/internalCommandsHandler.cpp | 75 +++++++++++++++---- .../TesterReplica/internalCommandsHandler.hpp | 16 ++-- tests/simpleKVBC/TesterReplica/main.cpp | 2 + 5 files changed, 83 insertions(+), 30 deletions(-) diff --git a/bftengine/src/bftengine/ReplicaImp.cpp b/bftengine/src/bftengine/ReplicaImp.cpp index f2be781467..78581e32a1 100644 --- a/bftengine/src/bftengine/ReplicaImp.cpp +++ b/bftengine/src/bftengine/ReplicaImp.cpp @@ -3076,6 +3076,17 @@ bool ReplicaImp::tryToEnterView() { return enteredView; } +size_t ReplicaImp::clearClientRequestQueue() { + size_t primaryCombinedReqSize = 0; + LOG_INFO(GL, "clearing client requests" << KVLOG(requestsQueueOfPrimary.size())); + // clear requestsQueueOfPrimary + while (!requestsQueueOfPrimary.empty()) { + primaryCombinedReqSize += requestsQueueOfPrimary.front()->size(); + requestsQueueOfPrimary.pop(); + } + return primaryCombinedReqSize; +} + void ReplicaImp::onNewView(std::vector> &prePreparesForNewView) { SCOPED_MDC_SEQ_NUM(std::to_string(getCurrentView())); SeqNum firstPPSeq = 0; @@ -3202,11 +3213,7 @@ void ReplicaImp::onNewView(std::vector> &prePrepa requestsOfNonPrimary.clear(); - // clear requestsQueueOfPrimary - while (!requestsQueueOfPrimary.empty()) { - primaryCombinedReqSize -= requestsQueueOfPrimary.front()->size(); - requestsQueueOfPrimary.pop(); - } + primaryCombinedReqSize -= clearClientRequestQueue(); primary_queue_size_.Get().Set(requestsQueueOfPrimary.size()); @@ -3299,6 +3306,8 @@ void ReplicaImp::onTransferringCompleteImp(uint64_t newStateCheckpoint) { time_in_state_transfer_.end(); LOG_INFO(GL, KVLOG(newStateCheckpoint)); requestsOfNonPrimary.clear(); + clearClientRequestQueue(); + if (ps_) { ps_->beginWriteTran(); } diff --git a/bftengine/src/bftengine/ReplicaImp.hpp b/bftengine/src/bftengine/ReplicaImp.hpp index d8b69e1165..d0e8921385 100644 --- a/bftengine/src/bftengine/ReplicaImp.hpp +++ b/bftengine/src/bftengine/ReplicaImp.hpp @@ -633,6 +633,7 @@ class ReplicaImp : public InternalReplicaApi, public ReplicaForStateTransfer { void addTimers(); void startConsensusProcess(PrePrepareMsgShPtr& pp, bool isCreatedEarlier); void startConsensusProcess(PrePrepareMsgShPtr& pp); + void clearClientRequestQueue(); /** * Updates both seqNumInfo and slow_path metric * @param seqNumInfo diff --git a/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp b/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp index 3629e9d291..66489fa5b7 100644 --- a/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp +++ b/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include "ReplicaConfig.hpp" #include "kvbc_key_types.hpp" @@ -64,6 +65,34 @@ static const std::string &keyHashToCategory(const Hash &keyHash) { static const std::string &keyToCategory(const std::string &key) { return keyHashToCategory(createHash(key)); } +InternalCommandsHandler::InternalCommandsHandler(concord::kvbc::IReader *storage, + concord::kvbc::IBlockAdder *blocksAdder, + concord::kvbc::IBlockMetadata *blockMetadata, + logging::Logger &logger, + bftEngine::IStateTransfer &st, + bool addAllKeysAsPublic, + concord::kvbc::adapter::ReplicaBlockchain *kvbc) + : m_storage(storage), + m_blockAdder(blocksAdder), + m_blockMetadata(blockMetadata), + m_logger(logger), + m_addAllKeysAsPublic{addAllKeysAsPublic}, + m_kvbc{kvbc} { + st.addOnTransferringCompleteCallback([this](uint64_t) { + LOG_INFO(GL, "Synchronizing client execution state after state transfer"); + auto data = m_storage->getLatest(CLIENT_STATE_CAT_ID, {0x1}); + ConcordAssert(data.has_value()); + auto raw_json = std::get(data.value()).data; + nlohmann::json json2 = nlohmann::json::parse(raw_json); + nlohmann::json::json_serializer, void> serializer; + serializer.from_json(json2, m_clientToMaxExecutedReqId); + LOG_INFO(GL, "raw client state: " << KVLOG(raw_json)); + }); + if (m_addAllKeysAsPublic) { + ConcordAssertNE(m_kvbc, nullptr); + } +} + void InternalCommandsHandler::add(std::string &&key, std::string &&value, VersionedUpdates &verUpdates, @@ -318,7 +347,15 @@ void InternalCommandsHandler::writeAccumulatedBlock(ExecutionRequestsQueue &bloc "SKVBCWrite message handled; writesCounter=" << m_writesCounter << " currBlock=" << write_rep.latest_block); } } - addBlock(verUpdates, merkleUpdates, sn); + + nlohmann::json json; + nlohmann::json::json_serializer, void> serializer; + serializer.to_json(json, m_clientToMaxExecutedReqId); + + VersionedUpdates clientStateUpdate; + clientStateUpdate.addUpdate({0x1}, json.dump()); + + addBlock(verUpdates, merkleUpdates, clientStateUpdate, sn); } OperationResult InternalCommandsHandler::verifyWriteCommand(uint32_t requestSize, @@ -364,7 +401,10 @@ void InternalCommandsHandler::addKeys(const SKVBCWriteRequest &writeReq, addMetadataKeyValue(verUpdates, sequenceNum); } -void InternalCommandsHandler::addBlock(VersionedUpdates &verUpdates, BlockMerkleUpdates &merkleUpdates, uint64_t sn) { +void InternalCommandsHandler::addBlock(VersionedUpdates &verUpdates, + BlockMerkleUpdates &merkleUpdates, + VersionedUpdates &clientStateVerUpdates, + uint64_t sn) { BlockId currBlock = m_storage->getLastBlockId(); Updates updates; @@ -396,6 +436,7 @@ void InternalCommandsHandler::addBlock(VersionedUpdates &verUpdates, BlockMerkle updates.add(kConcordInternalCategoryId, std::move(internal_updates)); updates.add(VERSIONED_KV_CAT_ID, std::move(verUpdates)); updates.add(BLOCK_MERKLE_CAT_ID, std::move(merkleUpdates)); + updates.add(CLIENT_STATE_CAT_ID, std::move(clientStateVerUpdates)); const auto newBlockId = m_blockAdder->add(std::move(updates)); ConcordAssert(newBlockId == currBlock + 1); } @@ -483,7 +524,17 @@ OperationResult InternalCommandsHandler::executeWriteCommand(uint32_t requestSiz hasConflict = hasConflict || (!isFirstClientRequest && m_clientToMaxExecutedReqId[clientId] >= requestId); } + SKVBCReply reply; + reply.reply = SKVBCWriteReply(); + SKVBCWriteReply &write_rep = std::get(reply.reply); + write_rep.success = !hasConflict; if (!hasConflict) { + write_rep.latest_block = currBlock + 1; + auto [iter, isNew] = m_clientToMaxExecutedReqId.emplace(clientId, 0); + UNUSED(iter); + UNUSED(isNew); + m_clientToMaxExecutedReqId[clientId] = std::max(m_clientToMaxExecutedReqId[clientId], batchCid); + if (isBlockAccumulationEnabled) { // If Block Accumulation is enabled then blocks are added after all requests are processed addKeys(write_req, sequenceNum, blockAccumulatedVerUpdates, blockAccumulatedMerkleUpdates); @@ -491,21 +542,17 @@ OperationResult InternalCommandsHandler::executeWriteCommand(uint32_t requestSiz // If Block Accumulation is not enabled then blocks are added after all requests are processed VersionedUpdates verUpdates; BlockMerkleUpdates merkleUpdates; + VersionedUpdates clientVerUpdates; + nlohmann::json json; + nlohmann::json::json_serializer, void> serializer; + serializer.to_json(json, m_clientToMaxExecutedReqId); + auto dump = json.dump(); + LOG_INFO(GL, KVLOG(dump)); + clientVerUpdates.addUpdate({0x1}, json.dump()); addKeys(write_req, sequenceNum, verUpdates, merkleUpdates); - addBlock(verUpdates, merkleUpdates, sequenceNum); + addBlock(verUpdates, merkleUpdates, clientVerUpdates, sequenceNum); } - } - SKVBCReply reply; - reply.reply = SKVBCWriteReply(); - SKVBCWriteReply &write_rep = std::get(reply.reply); - write_rep.success = !hasConflict; - if (!hasConflict) { - write_rep.latest_block = currBlock + 1; - auto [iter, isNew] = m_clientToMaxExecutedReqId.emplace(clientId, 0); - UNUSED(iter); - UNUSED(isNew); - m_clientToMaxExecutedReqId[clientId] = std::max(m_clientToMaxExecutedReqId[clientId], batchCid); } else { write_rep.latest_block = currBlock; } diff --git a/tests/simpleKVBC/TesterReplica/internalCommandsHandler.hpp b/tests/simpleKVBC/TesterReplica/internalCommandsHandler.hpp index 99f6ab2527..499b77d2b6 100644 --- a/tests/simpleKVBC/TesterReplica/internalCommandsHandler.hpp +++ b/tests/simpleKVBC/TesterReplica/internalCommandsHandler.hpp @@ -31,6 +31,8 @@ static const std::string VERSIONED_KV_CAT_ID{concord::kvbc::categorization::kExecutionPrivateCategory}; static const std::string BLOCK_MERKLE_CAT_ID{concord::kvbc::categorization::kExecutionProvableCategory}; +static constexpr const char *clientReplyStateCategory = "client_state"; +static const std::string CLIENT_STATE_CAT_ID{clientReplyStateCategory}; class InternalCommandsHandler : public concord::kvbc::ICommandsHandler { public: @@ -38,18 +40,9 @@ class InternalCommandsHandler : public concord::kvbc::ICommandsHandler { concord::kvbc::IBlockAdder *blocksAdder, concord::kvbc::IBlockMetadata *blockMetadata, logging::Logger &logger, + bftEngine::IStateTransfer &st, bool addAllKeysAsPublic = false, - concord::kvbc::adapter::ReplicaBlockchain *kvbc = nullptr) - : m_storage(storage), - m_blockAdder(blocksAdder), - m_blockMetadata(blockMetadata), - m_logger(logger), - m_addAllKeysAsPublic{addAllKeysAsPublic}, - m_kvbc{kvbc} { - if (m_addAllKeysAsPublic) { - ConcordAssertNE(m_kvbc, nullptr); - } - } + concord::kvbc::adapter::ReplicaBlockchain *kvbc = nullptr); void execute(ExecutionRequestsQueue &requests, std::optional timestamp, @@ -146,6 +139,7 @@ class InternalCommandsHandler : public concord::kvbc::ICommandsHandler { uint64_t sn); void addBlock(concord::kvbc::categorization::VersionedUpdates &verUpdates, concord::kvbc::categorization::BlockMerkleUpdates &merkleUpdates, + concord::kvbc::categorization::VersionedUpdates &clientStateUpdates, uint64_t sn); void addKeys(const skvbc::messages::SKVBCWriteRequest &writeReq, uint64_t sequenceNum, diff --git a/tests/simpleKVBC/TesterReplica/main.cpp b/tests/simpleKVBC/TesterReplica/main.cpp index 2b1de7a215..9a2edd9baf 100644 --- a/tests/simpleKVBC/TesterReplica/main.cpp +++ b/tests/simpleKVBC/TesterReplica/main.cpp @@ -130,6 +130,7 @@ void run_replica(int argc, char** argv) { setup->GetPerformanceManager(), std::map{ {VERSIONED_KV_CAT_ID, categorization::CATEGORY_TYPE::versioned_kv}, + {CLIENT_STATE_CAT_ID, categorization::CATEGORY_TYPE::versioned_kv}, {categorization::kExecutionEventGroupLatestCategory, categorization::CATEGORY_TYPE::versioned_kv}, {BLOCK_MERKLE_CAT_ID, categorization::CATEGORY_TYPE::block_merkle}}, setup->GetSecretManager()); @@ -151,6 +152,7 @@ void run_replica(int argc, char** argv) { replica.get(), blockMetadata, logger, + replica->getStateTransfer(), setup->AddAllKeysAsPublic(), replica->kvBlockchain() ? &replica->kvBlockchain().value() : nullptr); replica->set_command_handler(cmdHandler); From 04919244623950ad242fb65af36704ccfeb08239 Mon Sep 17 00:00:00 2001 From: Ynon Flum Date: Wed, 22 Feb 2023 17:25:46 +0200 Subject: [PATCH 07/18] Fix more errors --- .../bcstatetransfer/AsyncStateTransferCRE.cpp | 3 +- .../src/bftengine/ReplicaForStateTransfer.cpp | 3 + bftengine/src/bftengine/ReplicaImp.cpp | 10 +-- .../bftengine/messages/ClientRequestMsg.cpp | 2 +- .../poll_based_state_client.hpp | 5 +- .../src/poll_based_state_client.cpp | 15 ++++- .../TesterReplica/internalCommandsHandler.cpp | 65 +++++++++++++------ .../TesterReplica/internalCommandsHandler.hpp | 7 +- 8 files changed, 76 insertions(+), 34 deletions(-) diff --git a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp index 3afed4d5ab..c51fe6aab1 100644 --- a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp +++ b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp @@ -184,7 +184,8 @@ std::shared_ptr CreFactory::create( Config cre_config; cre_config.id_ = repConfig.replicaId; cre_config.interval_timeout_ms_ = 1000; - IStateClient* pbc = new PollBasedStateClient(bftClient, cre_config.interval_timeout_ms_, 0, cre_config.id_); + // TODO: fix relying on f + 1, so that byzantine replicas are also handled + IStateClient* pbc = new PollBasedStateClient(bftClient, cre_config.interval_timeout_ms_, 0, cre_config.id_, true); auto cre = std::make_shared(cre_config, pbc, std::make_shared()); if (bftEngine::ReplicaConfig::instance().isReadOnly) { diff --git a/bftengine/src/bftengine/ReplicaForStateTransfer.cpp b/bftengine/src/bftengine/ReplicaForStateTransfer.cpp index dea76c0e72..4d1c994aad 100644 --- a/bftengine/src/bftengine/ReplicaForStateTransfer.cpp +++ b/bftengine/src/bftengine/ReplicaForStateTransfer.cpp @@ -97,6 +97,7 @@ void ReplicaForStateTransfer::start() { // on other replicas but not on this one, finishing ST does not mean that missed key exchanges are executed) // This can be done by iterating the saved cryptosystems and updating their private key if their // public key matches the candidate saved in KeyExchangeManager + // TODO: persist the candidate CryptoManager::instance().onCheckpoint(checkpoint); auto [priv, pub] = KeyExchangeManager::instance().getCandidateKeyPair(); CryptoManager::instance().syncPrivateKeyAfterST(priv, pub); @@ -108,6 +109,8 @@ void ReplicaForStateTransfer::start() { auto *pbc = reinterpret_cast(cre_->getStateClient()); + // TODO: remove loop so that state transfer doesn't hang if it cannot complete reconfiguration requests + // The current implementation expects f + 1 identical responses bool succ = false; while (!succ) { auto latestHandledUpdate = cre_->getLatestKnownUpdateBlock(); diff --git a/bftengine/src/bftengine/ReplicaImp.cpp b/bftengine/src/bftengine/ReplicaImp.cpp index 78581e32a1..4a481a7ba5 100644 --- a/bftengine/src/bftengine/ReplicaImp.cpp +++ b/bftengine/src/bftengine/ReplicaImp.cpp @@ -2373,14 +2373,14 @@ void ReplicaImp::onMessage(std::unique_ptr message static uint32_t maxTimeSinceLastExecutionInMainWindowMs = config_.get("concord.bft.st.maxTimeSinceLastExecutionInMainWindowMs", 5000); - Time timeOfLastEcecution = MinTime; + Time timeOfLastExecution = MinTime; if (mainLog->insideActiveWindow(lastExecutedSeqNum)) - timeOfLastEcecution = mainLog->get(lastExecutedSeqNum).lastUpdateTimeOfCommitMsgs(); - if ((getMonotonicTime() - timeOfLastEcecution) > (milliseconds(maxTimeSinceLastExecutionInMainWindowMs))) { + timeOfLastExecution = mainLog->get(lastExecutedSeqNum).lastUpdateTimeOfCommitMsgs(); + if ((getMonotonicTime() - timeOfLastExecution) > (milliseconds(maxTimeSinceLastExecutionInMainWindowMs))) { LOG_INFO(GL, "Number of stable checkpoints in current window: " << numRelevant << " time since last execution: " - << (getMonotonicTime() - timeOfLastEcecution).count() << " ms"); + << (getMonotonicTime() - timeOfLastExecution).count() << " ms"); askForStateTransfer = true; startStReason = "Too much time has passed since last execution"; } @@ -2912,7 +2912,7 @@ void ReplicaImp::onMessage(std::unique_ptr message ViewNum maxKnownCorrectView = 0; ViewNum maxKnownAgreedView = 0; viewsManager->computeCorrectRelevantViewNumbers(&maxKnownCorrectView, &maxKnownAgreedView); - LOG_INFO(VC_LOG, "View Number details: " << KVLOG(maxKnownCorrectView, maxKnownAgreedView)); + LOG_INFO(VC_LOG, "View Number details: " << KVLOG(maxKnownCorrectView, maxKnownAgreedView, getCurrentView())); if (maxKnownCorrectView > getCurrentView()) { // we have at least f+1 view-changes with view number >= maxKnownCorrectView diff --git a/bftengine/src/bftengine/messages/ClientRequestMsg.cpp b/bftengine/src/bftengine/messages/ClientRequestMsg.cpp index 4be5db2e04..b46bbf7a32 100644 --- a/bftengine/src/bftengine/messages/ClientRequestMsg.cpp +++ b/bftengine/src/bftengine/messages/ClientRequestMsg.cpp @@ -143,7 +143,7 @@ void ClientRequestMsg::validateImp(const ReplicasInfo& repInfo) const { (repInfo.isIdOfReplica(clientId) || repInfo.isIdOfPeerRoReplica(clientId))) { // Allow every reconfiguration/internal message from replicas (it will be verified in the reconfiguration handler) LOG_INFO(CNSUS, - "Reconfig/Internal replica message not validated" + "Reconfig/Internal replica message validation skipped" << KVLOG(clientId, header->flags & RECONFIG_FLAG, header->flags & INTERNAL_FLAG)); return; } diff --git a/client/reconfiguration/include/client/reconfiguration/poll_based_state_client.hpp b/client/reconfiguration/include/client/reconfiguration/poll_based_state_client.hpp index 672c5378c1..c19e81ad1b 100644 --- a/client/reconfiguration/include/client/reconfiguration/poll_based_state_client.hpp +++ b/client/reconfiguration/include/client/reconfiguration/poll_based_state_client.hpp @@ -29,7 +29,8 @@ class PollBasedStateClient : public IStateClient { PollBasedStateClient(bft::client::Client* client, uint64_t interval_timeout_ms, uint64_t last_known_block, - const uint16_t id_); + const uint16_t id_, + bool use_byzantine_quorum = false); State getNextState() const override; bool updateState(const WriteState& state) override; ~PollBasedStateClient(); @@ -63,6 +64,8 @@ class PollBasedStateClient : public IStateClient { bool halted_ = false; std::condition_variable resume_cond_; std::mutex resume_lock_; + // At the end of State transfer we use a f + 1 quorum + bool use_byzantine_quorum_ = false; }; } // namespace concord::client::reconfiguration \ No newline at end of file diff --git a/client/reconfiguration/src/poll_based_state_client.cpp b/client/reconfiguration/src/poll_based_state_client.cpp index 4b12a93353..598bcdd2b6 100644 --- a/client/reconfiguration/src/poll_based_state_client.cpp +++ b/client/reconfiguration/src/poll_based_state_client.cpp @@ -29,7 +29,14 @@ concord::messages::ReconfigurationResponse PollBasedStateClient::sendReconfigura concord::messages::ReconfigurationResponse rres; try { if (read_request) { - bft::client::ReadConfig read_config{request_config, bft::client::LinearizableQuorum{}}; + // TODO: State transfer can work with f + 1 as long as there are no byzantine replicas + bft::client::ReadConfig read_config; + if (use_byzantine_quorum_) { + read_config = bft::client::ReadConfig{request_config, bft::client::ByzantineSafeQuorum{}}; + } else { + read_config = bft::client::ReadConfig{request_config, bft::client::LinearizableQuorum{}}; + } + rep = bftclient_->send(read_config, std::move(msg)); } else { bft::client::WriteConfig write_config{request_config, bft::client::LinearizableQuorum{}}; @@ -57,12 +64,14 @@ State PollBasedStateClient::getNextState() const { PollBasedStateClient::PollBasedStateClient(bft::client::Client* client, uint64_t interval_timeout_ms, uint64_t last_known_block, - const uint16_t id) + const uint16_t id, + bool use_byzantine_quorum) : bftclient_{client}, id_{id}, interval_timeout_ms_{interval_timeout_ms}, last_known_block_{last_known_block}, - sn_gen_(bft::client::ClientId{id}) {} + sn_gen_(bft::client::ClientId{id}), + use_byzantine_quorum_{use_byzantine_quorum} {} std::vector PollBasedStateClient::getStateUpdate(bool& succ) const { concord::messages::ClientReconfigurationStateRequest creq{id_}; diff --git a/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp b/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp index 66489fa5b7..686b276ddb 100644 --- a/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp +++ b/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp @@ -78,21 +78,39 @@ InternalCommandsHandler::InternalCommandsHandler(concord::kvbc::IReader *storage m_logger(logger), m_addAllKeysAsPublic{addAllKeysAsPublic}, m_kvbc{kvbc} { - st.addOnTransferringCompleteCallback([this](uint64_t) { - LOG_INFO(GL, "Synchronizing client execution state after state transfer"); - auto data = m_storage->getLatest(CLIENT_STATE_CAT_ID, {0x1}); - ConcordAssert(data.has_value()); - auto raw_json = std::get(data.value()).data; - nlohmann::json json2 = nlohmann::json::parse(raw_json); - nlohmann::json::json_serializer, void> serializer; - serializer.from_json(json2, m_clientToMaxExecutedReqId); - LOG_INFO(GL, "raw client state: " << KVLOG(raw_json)); - }); + if (ReplicaConfig::instance().isReadOnly) { + return; + } + + loadClientStateFromStorage(); + st.addOnTransferringCompleteCallback([this](uint64_t) { this->loadClientStateFromStorage(); }); + if (m_addAllKeysAsPublic) { ConcordAssertNE(m_kvbc, nullptr); } } +void InternalCommandsHandler::loadClientStateFromStorage() { + ConcordAssert(!ReplicaConfig::instance().isReadOnly); + LOG_INFO(GL, "Synchronizing client execution state"); + auto data = m_storage->getLatest(CLIENT_STATE_CAT_ID, {0x1}); + if (!data.has_value()) { + LOG_WARN(GL, "empty client execution state, were any client requests executed?"); + return; + } + auto raw_json = std::get(data.value()).data; + nlohmann::json json2 = nlohmann::json::parse(raw_json); + nlohmann::json::json_serializer>, void> serializer; + std::vector> deserialized; + serializer.from_json(json2, deserialized); + m_clientToMaxExecutedReqId.clear(); + + for (auto [clientId, reqId] : deserialized) { + m_clientToMaxExecutedReqId[clientId] = reqId; + } + LOG_INFO(GL, "raw client state: " << KVLOG(raw_json)); +} + void InternalCommandsHandler::add(std::string &&key, std::string &&value, VersionedUpdates &verUpdates, @@ -348,16 +366,26 @@ void InternalCommandsHandler::writeAccumulatedBlock(ExecutionRequestsQueue &bloc } } - nlohmann::json json; - nlohmann::json::json_serializer, void> serializer; - serializer.to_json(json, m_clientToMaxExecutedReqId); - VersionedUpdates clientStateUpdate; - clientStateUpdate.addUpdate({0x1}, json.dump()); + clientStateUpdate.addUpdate({0x1}, serializeClientState()); addBlock(verUpdates, merkleUpdates, clientStateUpdate, sn); } +std::string InternalCommandsHandler::serializeClientState() const { + nlohmann::json json; + nlohmann::json::json_serializer>, void> serializer; + // Need to maintain a fixed order in the blocks so that the replica state won't diverge + std::vector> serialized; + for (auto clientIdReqIdPair : m_clientToMaxExecutedReqId) { + serialized.push_back(clientIdReqIdPair); + } + serializer.to_json(json, serialized); + auto serialized_raw_json = json.dump(); + LOG_INFO(GL, KVLOG(serialized_raw_json)); + return json.dump(); +} + OperationResult InternalCommandsHandler::verifyWriteCommand(uint32_t requestSize, const uint8_t *request, size_t maxReplySize, @@ -543,12 +571,7 @@ OperationResult InternalCommandsHandler::executeWriteCommand(uint32_t requestSiz VersionedUpdates verUpdates; BlockMerkleUpdates merkleUpdates; VersionedUpdates clientVerUpdates; - nlohmann::json json; - nlohmann::json::json_serializer, void> serializer; - serializer.to_json(json, m_clientToMaxExecutedReqId); - auto dump = json.dump(); - LOG_INFO(GL, KVLOG(dump)); - clientVerUpdates.addUpdate({0x1}, json.dump()); + clientVerUpdates.addUpdate({0x1}, serializeClientState()); addKeys(write_req, sequenceNum, verUpdates, merkleUpdates); addBlock(verUpdates, merkleUpdates, clientVerUpdates, sequenceNum); } diff --git a/tests/simpleKVBC/TesterReplica/internalCommandsHandler.hpp b/tests/simpleKVBC/TesterReplica/internalCommandsHandler.hpp index 499b77d2b6..51b03d0395 100644 --- a/tests/simpleKVBC/TesterReplica/internalCommandsHandler.hpp +++ b/tests/simpleKVBC/TesterReplica/internalCommandsHandler.hpp @@ -22,7 +22,7 @@ #include "ControlStateManager.hpp" #include #include - +#include #include "log/logger.hpp" #include "skvbc_messages.cmf.hpp" #include "SharedTypes.hpp" @@ -150,6 +150,9 @@ class InternalCommandsHandler : public concord::kvbc::ICommandsHandler { concord::kvbc::categorization::VersionedUpdates &blockAccumulatedVerUpdates, concord::kvbc::categorization::BlockMerkleUpdates &blockAccumulatedMerkleUpdates) const; + std::string serializeClientState() const; + void loadClientStateFromStorage(); + private: concord::kvbc::IReader *m_storage; concord::kvbc::IBlockAdder *m_blockAdder; @@ -161,7 +164,7 @@ class InternalCommandsHandler : public concord::kvbc::ICommandsHandler { std::shared_ptr perfManager_; bool m_addAllKeysAsPublic{false}; // Add all key-values in the block merkle category as public ones. concord::kvbc::adapter::ReplicaBlockchain *m_kvbc{nullptr}; - std::unordered_map m_clientToMaxExecutedReqId; + std::map m_clientToMaxExecutedReqId; // This string is used by clients to distinguish blocks that should be ignored by them. // Some tests expect every block to be created by a request issued by test clients. From 1d7ab4ccd8a14385923f6da2d35316a945955dc1 Mon Sep 17 00:00:00 2001 From: Ynon Flum Date: Tue, 28 Feb 2023 16:58:41 +0200 Subject: [PATCH 08/18] Merge master [don't auto-bump] --- Makefile | 3 +- bftengine/include/bftengine/CryptoManager.hpp | 6 ++-- .../bcstatetransfer/AsyncStateTransferCRE.hpp | 1 + bftengine/src/bcstatetransfer/BCStateTran.cpp | 4 +-- bftengine/src/bftengine/CryptoManager.cpp | 3 -- .../src/bftengine/KeyExchangeManager.cpp | 4 +-- bftengine/src/bftengine/ReplicaBase.hpp | 12 ------- bftengine/src/bftengine/ReplicaImp.cpp | 2 +- bftengine/src/bftengine/ReplicaImp.hpp | 5 +-- bftengine/src/bftengine/SigManager.cpp | 35 ++++++++++--------- bftengine/src/bftengine/SigManager.hpp | 2 +- .../ValidationOnlyIdentityManager.cpp | 1 + .../ValidationOnlyIdentityManager.hpp | 3 +- .../messages/PreProcessResultMsg_test.cpp | 1 + .../preprocessor/tests/preprocessor_test.cpp | 3 +- .../clientsManager/ClientsManager_test.cpp | 1 + bftengine/tests/messages/helper.cpp | 2 +- bftengine/tests/messages/helper.hpp | 2 +- .../threshsign/ThresholdSignaturesTypes.cpp | 2 +- tests/apollo/test_skvbc_dbsnapshot.py | 3 -- tests/apollo/util/bft.py | 1 - tests/apollo/util/eliot_logging.py | 3 ++ tests/apollo/util/test_base.py | 1 + 23 files changed, 44 insertions(+), 56 deletions(-) diff --git a/Makefile b/Makefile index 805fa26c95..726e5e29ab 100644 --- a/Makefile +++ b/Makefile @@ -185,8 +185,7 @@ BASIC_RUN_PARAMS?=-it --init --rm --privileged=true \ --name="${CONCORD_BFT_DOCKER_CONTAINER}" \ --workdir=${CONCORD_BFT_TARGET_SOURCE_PATH} \ --mount type=bind,source=${CURDIR},target=${CONCORD_BFT_TARGET_SOURCE_PATH}${CONCORD_BFT_CONTAINER_MOUNT_CONSISTENCY} \ - ${CONCORD_BFT_ADDITIONAL_RUN_PARAMS} \ - ${CONCORD_BFT_DOCKER_IMAGE_FULL_PATH} + ${CONCORD_BFT_ADDITIONAL_RUN_PARAMS} ${CONCORD_BFT_DOCKER_IMAGE_FULL_PATH} .DEFAULT_GOAL:=build diff --git a/bftengine/include/bftengine/CryptoManager.hpp b/bftengine/include/bftengine/CryptoManager.hpp index e584e25372..f7d1c01a24 100644 --- a/bftengine/include/bftengine/CryptoManager.hpp +++ b/bftengine/include/bftengine/CryptoManager.hpp @@ -61,7 +61,7 @@ class CryptoManager : public IKeyExchanger, public IMultiSigKeyGenerator { // IKeyExchanger methods // onPrivateKeyExchange and onPublicKeyExchange callbacks for a given checkpoint may be called in a different order. - // Therefore the first called will create ca CryptoSys + // Therefore the first called will create a CryptoSys void onPrivateKeyExchange(const std::string& secretKey, const std::string& verificationKey, const SeqNum& sn) override; @@ -77,8 +77,8 @@ class CryptoManager : public IKeyExchanger, public IMultiSigKeyGenerator { * @param secretKey * @param verificationKey * @note: Assumes all keys are formatted as hex strings - * @note: TODO: Current implementation is not crash consistent, a ST which was completed after the replica process - * will lose the new private key + * @note: TODO: Current implementation is not crash consistent, a ST which was completed after the termination + * of the replica process will lose the new private key */ void syncPrivateKeyAfterST(const std::string& secretKey, const std::string& verificationKey); diff --git a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.hpp b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.hpp index dfb6735697..aaf5e06749 100644 --- a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.hpp +++ b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.hpp @@ -18,6 +18,7 @@ #include "MsgHandlersRegistrator.hpp" #include "MsgsCommunicator.hpp" #include "client/reconfiguration/client_reconfiguration_engine.hpp" +#include "crypto/signer.hpp" namespace bftEngine::bcst::asyncCRE { class CreFactory { diff --git a/bftengine/src/bcstatetransfer/BCStateTran.cpp b/bftengine/src/bcstatetransfer/BCStateTran.cpp index 29e9e684a0..107c905dde 100644 --- a/bftengine/src/bcstatetransfer/BCStateTran.cpp +++ b/bftengine/src/bcstatetransfer/BCStateTran.cpp @@ -3431,9 +3431,9 @@ void BCStateTran::processData(bool lastInBatch, uint32_t rvbDigestsSize) { LOG_ERROR(logger_, "Setting RVB digests into RVB manager failed!"); badDataFromCurrentSourceReplica = true; } else { - #ifdef ENABLE_ALL_METRICS +#ifdef ENABLE_ALL_METRICS metrics_.overall_rvb_digest_groups_validated_++; - #endif +#endif } } diff --git a/bftengine/src/bftengine/CryptoManager.cpp b/bftengine/src/bftengine/CryptoManager.cpp index 81216c225f..510026bd8a 100644 --- a/bftengine/src/bftengine/CryptoManager.cpp +++ b/bftengine/src/bftengine/CryptoManager.cpp @@ -10,7 +10,6 @@ // file. #include "ReplicaConfig.hpp" -#include "Logger.hpp" #include "CryptoManager.hpp" #include @@ -57,8 +56,6 @@ std::unique_ptr& CryptoManager::getLatestCryptoSystem() const { */ concord::crypto::SignatureAlgorithm CryptoManager::getLatestSignatureAlgorithm() const { const std::unordered_map typeToAlgorithm{ - {MULTISIG_BLS_SCHEME, concord::crypto::SignatureAlgorithm::BLS}, - {THRESHOLD_BLS_SCHEME, concord::crypto::SignatureAlgorithm::BLS}, {MULTISIG_EDDSA_SCHEME, concord::crypto::SignatureAlgorithm::EdDSA}, }; auto currentType = getLatestCryptoSystem()->getType(); diff --git a/bftengine/src/bftengine/KeyExchangeManager.cpp b/bftengine/src/bftengine/KeyExchangeManager.cpp index 77bca63157..3c4448ca29 100644 --- a/bftengine/src/bftengine/KeyExchangeManager.cpp +++ b/bftengine/src/bftengine/KeyExchangeManager.cpp @@ -89,9 +89,9 @@ void KeyExchangeManager::registerNewKeyPair(uint16_t repID, candidate_private_keys_.generated.clear(); // erasing seqnum from the map seq_candidate_map_.erase(sn); - ConcordAssert(private_keys_.key_data().generated.pub == kemsg.pubkey); + ConcordAssert(private_keys_.key_data().generated.pub == pubkey); private_keys_.onKeyExchange(cid, sn); - for (auto e : registryToExchange_) e->onPrivateKeyExchange(private_keys_.key_data().keys[sn], kemsg.pubkey, sn); + for (auto e : registryToExchange_) e->onPrivateKeyExchange(private_keys_.key_data().keys[sn], pubkey, sn); metrics_->self_key_exchange_counter++; } diff --git a/bftengine/src/bftengine/ReplicaBase.hpp b/bftengine/src/bftengine/ReplicaBase.hpp index 86cd8d0f81..4fb4406d24 100644 --- a/bftengine/src/bftengine/ReplicaBase.hpp +++ b/bftengine/src/bftengine/ReplicaBase.hpp @@ -99,18 +99,6 @@ class ReplicaBase { } } - /*bool validateMessage(MessageBase* msg) { - try { - if (config_.debugStatisticsEnabled) DebugStatistics::onReceivedExMessage(msg->type()); - - msg->validate(*repsInfo); - return true; - } catch (std::exception& e) { - onReportAboutInvalidMessage(msg, e.what()); - return false; - } - }*/ - protected: static const uint16_t ALL_OTHER_REPLICAS = UINT16_MAX; diff --git a/bftengine/src/bftengine/ReplicaImp.cpp b/bftengine/src/bftengine/ReplicaImp.cpp index 4a481a7ba5..990c4dd0f9 100644 --- a/bftengine/src/bftengine/ReplicaImp.cpp +++ b/bftengine/src/bftengine/ReplicaImp.cpp @@ -4439,7 +4439,7 @@ ReplicaImp::ReplicaImp(bool firstTime, concord::crypto::KeyFormat::PemFormat, {{repsInfo->getIdOfOperator(), ReplicaConfig::instance().getOperatorPublicKey(), - concord::crypto::KeyFormat::PemFormat}} + concord::crypto::KeyFormat::PemFormat}}, *repsInfo); viewsManager = new ViewsManager(repsInfo); } else { diff --git a/bftengine/src/bftengine/ReplicaImp.hpp b/bftengine/src/bftengine/ReplicaImp.hpp index d0e8921385..da74d89f53 100644 --- a/bftengine/src/bftengine/ReplicaImp.hpp +++ b/bftengine/src/bftengine/ReplicaImp.hpp @@ -371,9 +371,6 @@ class ReplicaImp : public InternalReplicaApi, public ReplicaForStateTransfer { void recoverRequests(); - template - bool validateMessage(MessageType* msg); - std::function getMessageValidator(); // InternalReplicaApi @@ -633,7 +630,7 @@ class ReplicaImp : public InternalReplicaApi, public ReplicaForStateTransfer { void addTimers(); void startConsensusProcess(PrePrepareMsgShPtr& pp, bool isCreatedEarlier); void startConsensusProcess(PrePrepareMsgShPtr& pp); - void clearClientRequestQueue(); + size_t clearClientRequestQueue(); /** * Updates both seqNumInfo and slow_path metric * @param seqNumInfo diff --git a/bftengine/src/bftengine/SigManager.cpp b/bftengine/src/bftengine/SigManager.cpp index 955e3d7444..4e22bf787b 100644 --- a/bftengine/src/bftengine/SigManager.cpp +++ b/bftengine/src/bftengine/SigManager.cpp @@ -49,14 +49,15 @@ SigManager* SigManager::instance() { void SigManager::reset(std::shared_ptr other) { s_sm = other; } -std::shared_ptr SigManager::init(ReplicaId myId, - const Key& mySigPrivateKey, - const std::set>& publicKeysOfReplicas, - KeyFormat replicasKeysFormat, - const std::set>>* publicKeysOfClients, - KeyFormat clientsKeysFormat, - const std::optional>& operatorKey, - const ReplicasInfo& replicasInfo) { +std::shared_ptr SigManager::init( + ReplicaId myId, + const Key& mySigPrivateKey, + const std::set>& publicKeysOfReplicas, + KeyFormat replicasKeysFormat, + const std::set>>* publicKeysOfClients, + KeyFormat clientsKeysFormat, + const std::optional>& operatorKey, + const ReplicasInfo& replicasInfo) { vector> publickeys; map publicKeysMapping; size_t lowBound, highBound; @@ -102,14 +103,14 @@ std::shared_ptr SigManager::init(ReplicaId myId, } LOG_INFO(GL, "Done Compute Start ctor for SigManager with " << KVLOG(publickeys.size(), publicKeysMapping.size())); - auto ret = std::shared_ptr{new SigManager( - myId, - make_pair(mySigPrivateKey, replicasKeysFormat), - publickeys, - publicKeysMapping, - ((ReplicaConfig::instance().clientTransactionSigningEnabled) && (publicKeysOfClients != nullptr)), - operatorKey, - replicasInfo)}; + auto ret = std::shared_ptr{ + new SigManager(myId, + make_pair(mySigPrivateKey, replicasKeysFormat), + publickeys, + publicKeysMapping, + ((ReplicaConfig::instance().clientTransactionSigningEnabled) && (publicKeysOfClients != nullptr)), + operatorKey, + replicasInfo)}; reset(ret); return ret; @@ -120,7 +121,7 @@ SigManager::SigManager(PrincipalId myId, const vector>& publickeys, const map& publicKeysMapping, bool clientTransactionSigningEnabled, - const std::optional>& operatorKey + const std::optional>& operatorKey, const ReplicasInfo& replicasInfo) : myId_(myId), clientTransactionSigningEnabled_(clientTransactionSigningEnabled), diff --git a/bftengine/src/bftengine/SigManager.hpp b/bftengine/src/bftengine/SigManager.hpp index 96de95ca60..9fd5c2044b 100644 --- a/bftengine/src/bftengine/SigManager.hpp +++ b/bftengine/src/bftengine/SigManager.hpp @@ -14,10 +14,10 @@ #include "PrimitiveTypes.hpp" #include "util/assertUtils.hpp" #include "util/Metrics.hpp" +#include "util/memory.hpp" #include "crypto/crypto.hpp" #include "crypto/signer.hpp" #include "crypto/verifier.hpp" -#include "util/memory.hpp" #include "SysConsts.hpp" #include #include diff --git a/bftengine/src/bftengine/ValidationOnlyIdentityManager.cpp b/bftengine/src/bftengine/ValidationOnlyIdentityManager.cpp index b8617382d9..cbec778b5e 100644 --- a/bftengine/src/bftengine/ValidationOnlyIdentityManager.cpp +++ b/bftengine/src/bftengine/ValidationOnlyIdentityManager.cpp @@ -24,6 +24,7 @@ ValidationOnlyIdentityManager::ValidationOnlyIdentityManager( publickeys, publicKeysMapping, false, + {}, replicasInfo) {} bool ValidationOnlyIdentityManager::verifySig( diff --git a/bftengine/src/bftengine/ValidationOnlyIdentityManager.hpp b/bftengine/src/bftengine/ValidationOnlyIdentityManager.hpp index 878251eefe..ace254a3e4 100644 --- a/bftengine/src/bftengine/ValidationOnlyIdentityManager.hpp +++ b/bftengine/src/bftengine/ValidationOnlyIdentityManager.hpp @@ -17,7 +17,8 @@ namespace bftEngine::impl { /* This class is a hack to enable the validation of replica signatures using fixed keys without initializing a CryptoManager object. Messages such as CheckpointMsg use a global singleton to expose their validation logic, - thus an instance of this class can be used to succeed in performing the validation. + thus an instance of this class can be registered as a global SigManager to succeed in performing CheckpointMsg + validations. */ class ValidationOnlyIdentityManager : public SigManager { public: diff --git a/bftengine/src/preprocessor/tests/messages/PreProcessResultMsg_test.cpp b/bftengine/src/preprocessor/tests/messages/PreProcessResultMsg_test.cpp index cd8fef6717..549e3d7340 100644 --- a/bftengine/src/preprocessor/tests/messages/PreProcessResultMsg_test.cpp +++ b/bftengine/src/preprocessor/tests/messages/PreProcessResultMsg_test.cpp @@ -58,6 +58,7 @@ std::shared_ptr createSigManagerWithTransactionSign replicasKeysFormat, publicKeysOfClients, concord::crypto::KeyFormat::HexaDecimalStrippedFormat, + {}, replicasInfo); } diff --git a/bftengine/src/preprocessor/tests/preprocessor_test.cpp b/bftengine/src/preprocessor/tests/preprocessor_test.cpp index 0ede21dddd..93c70f58a3 100644 --- a/bftengine/src/preprocessor/tests/preprocessor_test.cpp +++ b/bftengine/src/preprocessor/tests/preprocessor_test.cpp @@ -23,7 +23,7 @@ #include "ReplicaConfig.hpp" #include "IncomingMsgsStorageImp.hpp" #include "gtest/gtest.h" -#include "threshsign/eddsa/EdDSAMultisigFactory.h" +#include "crypto/threshsign/eddsa/EdDSAMultisigFactory.h" #include "CryptoManager.hpp" #include "tests/messages/helper.hpp" #include "tests/config/test_comm_config.hpp" @@ -216,6 +216,7 @@ void setUpConfiguration_4() { concord::crypto::KeyFormat::HexaDecimalStrippedFormat, nullptr, concord::crypto::KeyFormat::HexaDecimalStrippedFormat, + {}, *replicasInfo[i].get()); cryptoManager[i] = CryptoManager::init(std::make_unique(i, publicKeysVector, replicaPrivKeys.at(i))); diff --git a/bftengine/tests/clientsManager/ClientsManager_test.cpp b/bftengine/tests/clientsManager/ClientsManager_test.cpp index a016ae3ffd..68ba9f1e4e 100644 --- a/bftengine/tests/clientsManager/ClientsManager_test.cpp +++ b/bftengine/tests/clientsManager/ClientsManager_test.cpp @@ -152,6 +152,7 @@ static void resetSigManager() { kKeyFormatForTesting, &kInitialPublicKeysOfClientsForTesting, kKeyFormatForTesting, + {}, *sigManagerReplicasInfoForTesting); } diff --git a/bftengine/tests/messages/helper.cpp b/bftengine/tests/messages/helper.cpp index 0e01a16c40..d7e75f6f17 100644 --- a/bftengine/tests/messages/helper.cpp +++ b/bftengine/tests/messages/helper.cpp @@ -84,7 +84,7 @@ class TestSigManager : public bftEngine::impl::SigManager { const ReplicasInfo& replicasInfo, bool transactionSigningEnabled = false) : bftEngine::impl::SigManager( - myId, mySigPrivateKey, publickeys, publicKeysMapping, transactionSigningEnabled, replicasInfo) {} + myId, mySigPrivateKey, publickeys, publicKeysMapping, transactionSigningEnabled, {}, replicasInfo) {} static std::shared_ptr init(size_t myId, std::string& myPrivateKey, diff --git a/bftengine/tests/messages/helper.hpp b/bftengine/tests/messages/helper.hpp index 48eefcc25a..da31d405a9 100644 --- a/bftengine/tests/messages/helper.hpp +++ b/bftengine/tests/messages/helper.hpp @@ -26,7 +26,7 @@ #include "crypto/threshsign/IPublicKey.h" #include "CryptoManager.hpp" #include "SigManager.hpp" -#include "threshsign/eddsa/EdDSAMultisigFactory.h" +#include "crypto/threshsign/eddsa/EdDSAMultisigFactory.h" using bftEngine::impl::ReplicasInfo; diff --git a/libs/crypto/src/threshsign/ThresholdSignaturesTypes.cpp b/libs/crypto/src/threshsign/ThresholdSignaturesTypes.cpp index 57c64abe35..a10d06f28b 100644 --- a/libs/crypto/src/threshsign/ThresholdSignaturesTypes.cpp +++ b/libs/crypto/src/threshsign/ThresholdSignaturesTypes.cpp @@ -33,7 +33,7 @@ Cryptosystem::Cryptosystem(const std::string& sysType, subtype_(sysSubtype), numSigners_(sysNumSigners), threshold_(sysThreshold), - signerID_(NID), + signerID_(INVALID_SIGNER_ID), publicKey_("uninitialized") { if (!isValidCryptosystemSelection(sysType, sysSubtype, sysNumSigners, sysThreshold)) { throw std::runtime_error( diff --git a/tests/apollo/test_skvbc_dbsnapshot.py b/tests/apollo/test_skvbc_dbsnapshot.py index ad3d591def..d8151d60a7 100644 --- a/tests/apollo/test_skvbc_dbsnapshot.py +++ b/tests/apollo/test_skvbc_dbsnapshot.py @@ -595,9 +595,6 @@ async def test_db_checkpoint_creation_with_wedge(self, bft_network): op = operator.Operator( bft_network.config, client, bft_network.builddir) await op.wedge() - #from time import sleep - #sleep(1) - #return await bft_network.wait_for_stable_checkpoint( bft_network.all_replicas(), stable_seqnum = (checkpoint_before + 2) * 150) await self.validate_stop_on_wedge_point(bft_network, skvbc=skvbc, fullWedge=True) # verify that snapshot is created on wedge point diff --git a/tests/apollo/util/bft.py b/tests/apollo/util/bft.py index b102332e1a..78d3ab4f45 100644 --- a/tests/apollo/util/bft.py +++ b/tests/apollo/util/bft.py @@ -623,7 +623,6 @@ def _create_clients(self): def _create_new_client(self, client_class, client_id): config = self._bft_config(client_id) - log_message(message_type=f"Creating client {client_id}", config=str(config)) ro_replicas = [r.id for r in self.ro_replicas] return client_class(config, self.replicas, self.background_nursery, ro_replicas=ro_replicas) diff --git a/tests/apollo/util/eliot_logging.py b/tests/apollo/util/eliot_logging.py index 09b7e407fb..fc626c9f67 100644 --- a/tests/apollo/util/eliot_logging.py +++ b/tests/apollo/util/eliot_logging.py @@ -34,6 +34,9 @@ def set_file_destination(): now = logdir_timestamp() test_name = f"apollo_run_{now}" + if os.environ.get('BLOCKCHAIN_VERSION', default="1").lower() == "4": + test_name = test_name + "_v4" + relative_apollo_logs = 'tests/apollo/logs' relative_current_run_logs = f'{relative_apollo_logs}/{logdir_timestamp()}' logs_dir = f'../../build/{relative_current_run_logs}' diff --git a/tests/apollo/util/test_base.py b/tests/apollo/util/test_base.py index e7669cb3cf..24ce5f10a2 100644 --- a/tests/apollo/util/test_base.py +++ b/tests/apollo/util/test_base.py @@ -87,6 +87,7 @@ async def wrapper(*args, **kwargs): return decorator + def repeat_test(max_repeats: int, break_on_first_failure: bool, break_on_first_success: bool, test_name=None): """ Runs a test max_repeats times when both break_on_first_failure and break_on_first_success et to False. From f10a65147a41ffe698b5a849d735d07ba6022075 Mon Sep 17 00:00:00 2001 From: Ynon Flum Date: Sun, 5 Mar 2023 14:08:58 +0200 Subject: [PATCH 09/18] Address CR comments --- .../include/bftengine/KeyExchangeManager.hpp | 1 + .../bcstatetransfer/AsyncStateTransferCRE.cpp | 3 --- bftengine/src/bftengine/CryptoManager.cpp | 1 - bftengine/src/bftengine/KeyStore.cpp | 13 ------------- bftengine/src/bftengine/KeyStore.h | 1 - bftengine/src/bftengine/Reconfiguration.cpp | 4 +++- bftengine/src/bftengine/ReplicaBase.hpp | 7 +++++++ .../src/bftengine/ReplicaForStateTransfer.cpp | 13 ++++--------- bftengine/src/bftengine/RequestHandler.cpp | 18 +++++++++--------- bftengine/src/bftengine/SigManager.cpp | 13 +++++++------ .../src/bftengine/messages/CheckpointMsg.cpp | 2 +- .../messages/ReplicaAsksToLeaveViewMsg.cpp | 2 +- .../messages/ReplicaRestartReadyMsg.cpp | 2 +- .../src/bftengine/messages/ViewChangeMsg.cpp | 2 +- .../messages/PreProcessReplyMsg.cpp | 2 +- libs/crypto/openssl/EdDSASigner.hpp | 2 -- .../threshsign/ThresholdSignaturesTypes.cpp | 14 +++++++++----- .../threshsign/eddsa/EdDSAMultisigSigner.cpp | 3 +-- tests/apollo/util/bft.py | 2 +- .../TesterReplica/internalCommandsHandler.cpp | 8 +++++--- 20 files changed, 52 insertions(+), 61 deletions(-) diff --git a/bftengine/include/bftengine/KeyExchangeManager.hpp b/bftengine/include/bftengine/KeyExchangeManager.hpp index f374e163aa..acba50c06d 100644 --- a/bftengine/include/bftengine/KeyExchangeManager.hpp +++ b/bftengine/include/bftengine/KeyExchangeManager.hpp @@ -32,6 +32,7 @@ class KeyExchangeManager { public: void exchangeTlsKeys(const SeqNum& bft_sn); // Generates and publish key to consensus + // TODO: persist the candidate void generateConsensusKeyAndSendInternalClientMsg(const SeqNum& sn); // Send the current main public key of the replica to consensus void sendMainPublicKey(); diff --git a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp index c51fe6aab1..276dfe8a3c 100644 --- a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp +++ b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp @@ -72,8 +72,6 @@ class Communication : public ICommunication { uint16_t repId_; }; -// Implement cryptomanager update handler - class ScalingReplicaHandler : public IStateHandler { public: ScalingReplicaHandler() {} @@ -130,7 +128,6 @@ class ScalingReplicaHandler : public IStateHandler { } }; -// TODO(yf): remove class MainKeyUpdateHandler : public IStateHandler { public: MainKeyUpdateHandler() { LOG_INFO(getLogger(), "Created StateTransfer CRE replica main key update handler"); } diff --git a/bftengine/src/bftengine/CryptoManager.cpp b/bftengine/src/bftengine/CryptoManager.cpp index 510026bd8a..c32d6f086b 100644 --- a/bftengine/src/bftengine/CryptoManager.cpp +++ b/bftengine/src/bftengine/CryptoManager.cpp @@ -218,7 +218,6 @@ std::shared_ptr CryptoManager::create(const LOG_INFO(logger(), "Created new cryptosystem for checkpoint: " << chckp << ", insertion success: " << insert_result.second << KVLOG(cryptoSystems_.size())); - assertMapSizeValid(); return ret; } diff --git a/bftengine/src/bftengine/KeyStore.cpp b/bftengine/src/bftengine/KeyStore.cpp index 31673c3255..1f9b3fc79c 100644 --- a/bftengine/src/bftengine/KeyStore.cpp +++ b/bftengine/src/bftengine/KeyStore.cpp @@ -39,19 +39,6 @@ std::optional ClusterKeyStore::loadReplicaKeyStoreF std::istringstream iss(buffer_); PublicKeys ks; PublicKeys::deserialize(iss, ks); - // std::vector keysToRemove; - for (auto [sn, pk] : ks.keys) { - /*if (ks.keys.size() - keysToRemove.size() > 2) { - keysToRemove.push_back(sn); - }*/ - LOG_INFO(KEY_EX_LOG, "Deserialized public key from reserved pages: " << KVLOG(repID, sn, pk)); - } - - /*for (auto sn : keysToRemove) { - LOG_INFO(KEY_EX_LOG, "Removing stale key from keystore" << KVLOG(sn, ks.keys[sn])); - ks.remove(sn); - }*/ - return ks; } catch (const std::exception& e) { LOG_FATAL(KEY_EX_LOG, diff --git a/bftengine/src/bftengine/KeyStore.h b/bftengine/src/bftengine/KeyStore.h index d72c81fef3..805d648c73 100644 --- a/bftengine/src/bftengine/KeyStore.h +++ b/bftengine/src/bftengine/KeyStore.h @@ -35,7 +35,6 @@ class ClusterKeyStore : public ResPagesClient { auto res = keys.insert(std::make_pair(sn, pub)); if (!res.second) ConcordAssert(pub == res.first->second) // if existed expect same key } - void remove(const SeqNum sn) { ConcordAssertEQ(keys.erase(sn), 1); } void serializeDataMembers(std::ostream& outStream) const { serialize(outStream, keys); } void deserializeDataMembers(std::istream& inStream) { deserialize(inStream, keys); } std::map keys; diff --git a/bftengine/src/bftengine/Reconfiguration.cpp b/bftengine/src/bftengine/Reconfiguration.cpp index eb520dfd33..7826ec3f27 100644 --- a/bftengine/src/bftengine/Reconfiguration.cpp +++ b/bftengine/src/bftengine/Reconfiguration.cpp @@ -34,7 +34,9 @@ bool ReconfigurationHandler::handle(const WedgeCommand& cmd, const std::optional&, concord::messages::ReconfigurationResponse&) { LOG_INFO(getLogger(), - "Wedge command instructs replica to stop at next checkpoint after sequence number " << bft_seq_num); + "Wedge command instructs replica to stop after to checkpoints " + "(after sequence number " + << bft_seq_num << ")"); bftEngine::ControlStateManager::instance().setStopAtNextCheckpoint(bft_seq_num); if (cmd.noop == false) addCreateDbSnapshotCbOnWedge(true); diff --git a/bftengine/src/bftengine/ReplicaBase.hpp b/bftengine/src/bftengine/ReplicaBase.hpp index 4fb4406d24..4385d1d163 100644 --- a/bftengine/src/bftengine/ReplicaBase.hpp +++ b/bftengine/src/bftengine/ReplicaBase.hpp @@ -87,6 +87,13 @@ class ReplicaBase { DebugStatistics::onReceivedExMessage(msg->type()); } try { + /* Checkpoint messages are used by replicas to initiate state transfer. + * Since a replica which initiates state transfer might not have the most + * public keys of other replicas, it might not be able to validate the signature + * of checkpoint messages. State transfer only requires that a replica has a valid + * communication channel. + * Therefore, replicas don't validate signatures of checkpoint messages. + */ if constexpr (std::is_same_v) { msg->validate(*repsInfo, false); } else { diff --git a/bftengine/src/bftengine/ReplicaForStateTransfer.cpp b/bftengine/src/bftengine/ReplicaForStateTransfer.cpp index 4d1c994aad..29a44f71c1 100644 --- a/bftengine/src/bftengine/ReplicaForStateTransfer.cpp +++ b/bftengine/src/bftengine/ReplicaForStateTransfer.cpp @@ -73,11 +73,6 @@ ReplicaForStateTransfer::ReplicaForStateTransfer(const ReplicaConfig &config, } void ReplicaForStateTransfer::start() { - /* TODO: Transaction signing needs to be disabled for replicas, as state transfer may change - * the a replica's consensus (and therefore main) key. Apollo tests currently run with transaction signing enabled, - * we therefore use the latest key published by this replica. - * If a key exchange consensus was reached by this replica, the other replicas are guaranteed to be able to use - * its latest key upon honest execution of the exchange */ cre_ = bftEngine::bcst::asyncCRE::CreFactory::create( msgsCommunicator_, msgHandlers_, std::make_unique()); stateTransfer->setReconfigurationEngine(cre_); @@ -97,7 +92,6 @@ void ReplicaForStateTransfer::start() { // on other replicas but not on this one, finishing ST does not mean that missed key exchanges are executed) // This can be done by iterating the saved cryptosystems and updating their private key if their // public key matches the candidate saved in KeyExchangeManager - // TODO: persist the candidate CryptoManager::instance().onCheckpoint(checkpoint); auto [priv, pub] = KeyExchangeManager::instance().getCandidateKeyPair(); CryptoManager::instance().syncPrivateKeyAfterST(priv, pub); @@ -123,11 +117,12 @@ void ReplicaForStateTransfer::start() { succ = false; break; } // else if (!isGettingBlocks) - // 2. Now we can safely halt cre. We know for sure that there are no update in the state transferred - // blocks that haven't been handled yet - cre_->halt(); } } // while (!succ) + + // 2. Now we can safely halt cre. We know for sure that there are no update in the state transferred + // blocks that haven't been handled yet + cre_->halt(); } }, IStateTransfer::StateTransferCallBacksPriorities::HIGH); diff --git a/bftengine/src/bftengine/RequestHandler.cpp b/bftengine/src/bftengine/RequestHandler.cpp index 3e0da5173b..8cade7bd6d 100644 --- a/bftengine/src/bftengine/RequestHandler.cpp +++ b/bftengine/src/bftengine/RequestHandler.cpp @@ -56,14 +56,14 @@ void RequestHandler::execute(IRequestsHandler::ExecutionRequestsQueue& requests, concordUtils::SpanWrapper& parent_span) { bool has_pruning_request = false; for (auto& req : requests) { - LOG_INFO(GL, - "Executing request: " << KVLOG(req.clientId, - req.cid, - req.flags, - req.requestSequenceNum, - req.signature.size(), - req.executionSequenceNum, - req.requestSize)); + LOG_DEBUG(GL, + "Executing request: " << KVLOG(req.clientId, + req.cid, + req.flags, + req.requestSequenceNum, + req.signature.size(), + req.executionSequenceNum, + req.requestSize)); if (req.flags & KEY_EXCHANGE_FLAG) { KeyExchangeMsg ke = KeyExchangeMsg::deserializeMsg(req.request, req.requestSize); LOG_INFO(KEY_EX_LOG, "BFT handler received KEY_EXCHANGE msg " << ke.toString()); @@ -80,7 +80,7 @@ void RequestHandler::execute(IRequestsHandler::ExecutionRequestsQueue& requests, } else if (req.flags & MsgFlag::RECONFIG_FLAG) { ReconfigurationRequest rreq; deserialize(std::vector(req.request, req.request + req.requestSize), rreq); - LOG_INFO( + LOG_DEBUG( GL, "Executing Reconfig request: " << KVLOG(rreq.signature.size(), rreq.command.index(), rreq.sender, rreq.id)); has_pruning_request = std::holds_alternative(rreq.command); diff --git a/bftengine/src/bftengine/SigManager.cpp b/bftengine/src/bftengine/SigManager.cpp index 4e22bf787b..305bb4b0da 100644 --- a/bftengine/src/bftengine/SigManager.cpp +++ b/bftengine/src/bftengine/SigManager.cpp @@ -268,10 +268,7 @@ size_t SigManager::sign(SeqNum seq, const concord::Byte* data, size_t dataLength rawSigner = mySigner_.get(); } auto result = rawSigner->signBuffer(data, dataLength, outSig); - // TODO(yf): remove private key and change to debug - LOG_DEBUG( - GL, - "Signing as replica with " << KVLOG(myId_, seq, rawSigner->getPrivKey(), rawSigner->signatureLength(), result)); + LOG_DEBUG(GL, "Signing as replica with " << KVLOG(myId_, seq, rawSigner->signatureLength(), result)); return result; } @@ -388,11 +385,15 @@ std::shared_ptr SigManager::getLastReplicaSigner() const { } std::pair SigManager::getMyLatestPublicKey() const { - ConcordAssert(ReplicaConfig::instance().singleSignatureScheme); ConcordAssert(replicasInfo_.isIdOfReplica(myId_)); + + if (!ReplicaConfig::instance().singleSignatureScheme) { + return {0, getVerifier(myId_).getPubKey()}; + } + auto [seq, latestVerifier] = CryptoManager::instance().getLatestVerifiers()[0]; auto pubKey = latestVerifier->getVerifier(myId_).getPubKey(); - LOG_DEBUG(GL, KVLOG(pubKey)); + LOG_DEBUG(GL, KVLOG(seq, pubKey)); return {seq, pubKey}; } diff --git a/bftengine/src/bftengine/messages/CheckpointMsg.cpp b/bftengine/src/bftengine/messages/CheckpointMsg.cpp index aeb2dd09b2..3a8a257c82 100644 --- a/bftengine/src/bftengine/messages/CheckpointMsg.cpp +++ b/bftengine/src/bftengine/messages/CheckpointMsg.cpp @@ -81,7 +81,7 @@ void CheckpointMsg::validate(const ReplicasInfo& repInfo, bool validateSignature validateSize(repInfo); if (validateSignature && !SigManager::instance()->verifySig(idOfGeneratedReplica(), getDataBytes(), getSignatureBytes())) { - throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": verifySig(replica)")); + throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": verifySig")); } } diff --git a/bftengine/src/bftengine/messages/ReplicaAsksToLeaveViewMsg.cpp b/bftengine/src/bftengine/messages/ReplicaAsksToLeaveViewMsg.cpp index 83049c1291..c1f5a9c7a4 100644 --- a/bftengine/src/bftengine/messages/ReplicaAsksToLeaveViewMsg.cpp +++ b/bftengine/src/bftengine/messages/ReplicaAsksToLeaveViewMsg.cpp @@ -64,7 +64,7 @@ void ReplicaAsksToLeaveViewMsg::validate(const ReplicasInfo& repInfo) const { if (!sigManager->verifySig(idOfGeneratedReplica(), std::string_view{body(), sizeof(Header)}, std::string_view{body() + totalSize, sigLen})) - throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": verifySigAsReplica")); + throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": verifySig")); } } // namespace impl diff --git a/bftengine/src/bftengine/messages/ReplicaRestartReadyMsg.cpp b/bftengine/src/bftengine/messages/ReplicaRestartReadyMsg.cpp index 55a7152b3d..38d1d8caca 100644 --- a/bftengine/src/bftengine/messages/ReplicaRestartReadyMsg.cpp +++ b/bftengine/src/bftengine/messages/ReplicaRestartReadyMsg.cpp @@ -80,7 +80,7 @@ void ReplicaRestartReadyMsg::validate(const ReplicasInfo& repInfo) const { if (!sigManager->verifySig( idOfSenderReplica, std::string_view{body(), dataSize}, std::string_view{body() + dataSize, sigLen})) - throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": verifySig(replica)")); + throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": verifySig")); } } // namespace impl diff --git a/bftengine/src/bftengine/messages/ViewChangeMsg.cpp b/bftengine/src/bftengine/messages/ViewChangeMsg.cpp index 8379e0c6ea..254a2ec8ff 100644 --- a/bftengine/src/bftengine/messages/ViewChangeMsg.cpp +++ b/bftengine/src/bftengine/messages/ViewChangeMsg.cpp @@ -183,7 +183,7 @@ void ViewChangeMsg::validate(const ReplicasInfo& repInfo) const { if (size() < (dataLength + sigLen)) throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": size")); if (!sigManager->verifySig( idOfGeneratedReplica(), std::string_view{body(), dataLength}, std::string_view{body() + dataLength, sigLen})) - throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": verifySig(replica)")); + throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": verifySig")); if (!checkElements(sigLen)) // check elements in message throw std::runtime_error(__PRETTY_FUNCTION__ + std::string(": check elements in message")); if (!checkComplaints(sigLen)) // check list of complaints diff --git a/bftengine/src/preprocessor/messages/PreProcessReplyMsg.cpp b/bftengine/src/preprocessor/messages/PreProcessReplyMsg.cpp index 9f9013ddf1..2c294afc78 100644 --- a/bftengine/src/preprocessor/messages/PreProcessReplyMsg.cpp +++ b/bftengine/src/preprocessor/messages/PreProcessReplyMsg.cpp @@ -87,7 +87,7 @@ void PreProcessReplyMsg::validate(const ReplicasInfo& repInfo) const { concord::crypto::SHA3_256::SIZE_IN_BYTES, reinterpret_cast(msgBody()) + headerSize, sigLen)) - throw runtime_error(__PRETTY_FUNCTION__ + string(": verifySig(replica) failed")); + throw runtime_error(__PRETTY_FUNCTION__ + string(": verifySig failed")); } } // namespace preprocessor diff --git a/libs/crypto/openssl/EdDSASigner.hpp b/libs/crypto/openssl/EdDSASigner.hpp index fbd9345823..7d13937950 100644 --- a/libs/crypto/openssl/EdDSASigner.hpp +++ b/libs/crypto/openssl/EdDSASigner.hpp @@ -36,8 +36,6 @@ class EdDSASigner : public ISigner { explicit EdDSASigner(const PrivateKeyType &privateKey) : privateKey_(privateKey) {} size_t signBuffer(const concord::Byte *msg, size_t len, concord::Byte *signature) const override { - // TODO: remove - LOG_DEBUG(GL, "Signing with key " << getPrivKey()); UniquePKEY pkey(EVP_PKEY_new_raw_private_key( NID_ED25519, nullptr, privateKey_.getBytes().data(), privateKey_.getBytes().size())); ConcordAssertNE(pkey, nullptr); diff --git a/libs/crypto/src/threshsign/ThresholdSignaturesTypes.cpp b/libs/crypto/src/threshsign/ThresholdSignaturesTypes.cpp index a10d06f28b..dc1001b9df 100644 --- a/libs/crypto/src/threshsign/ThresholdSignaturesTypes.cpp +++ b/libs/crypto/src/threshsign/ThresholdSignaturesTypes.cpp @@ -126,13 +126,17 @@ std::string Cryptosystem::getPrivateKey(uint16_t signerIndex) const { throw std::out_of_range(__PRETTY_FUNCTION__ + std::string(" Signer index out of range: ") + std::to_string(signerIndex)); - if (privateKeys_.size() < 1) { - throw std::runtime_error( - "Private keys have not been" - " generated or loaded for this cryptosystem."); + if (privateKeys_.size() == 1) { + return privateKeys_[0]; } - return privateKeys_[signerIndex]; + if (privateKeys_.size() == numSigners_) { + return privateKeys_[signerIndex]; + } + + throw std::runtime_error( + "Private keys have not been" + " generated or loaded for this cryptosystem."); } void Cryptosystem::loadKeys(const std::string& publicKey, const std::vector& verificationKeys) { diff --git a/libs/crypto/src/threshsign/eddsa/EdDSAMultisigSigner.cpp b/libs/crypto/src/threshsign/eddsa/EdDSAMultisigSigner.cpp index 3f4eedfe5e..319ed8fd79 100644 --- a/libs/crypto/src/threshsign/eddsa/EdDSAMultisigSigner.cpp +++ b/libs/crypto/src/threshsign/eddsa/EdDSAMultisigSigner.cpp @@ -17,8 +17,7 @@ using concord::crypto::openssl::EdDSASigner; EdDSAMultisigSigner::EdDSAMultisigSigner(const EdDSAThreshsignPrivateKey &privateKey, const uint32_t id) : EdDSASigner{privateKey}, publicKey_{}, id_{id} { - // TODO: remove log message - LOG_DEBUG(EDDSA_MULTISIG_LOG, "created eddsa signer with " << KVLOG(id_, getPrivKey())); + LOG_DEBUG(EDDSA_MULTISIG_LOG, "created eddsa signer with " << KVLOG(id_)); } int EdDSAMultisigSigner::requiredLengthForSignedData() const { return sizeof(SingleEdDSASignature); } diff --git a/tests/apollo/util/bft.py b/tests/apollo/util/bft.py index 78d3ab4f45..b73f6529fe 100644 --- a/tests/apollo/util/bft.py +++ b/tests/apollo/util/bft.py @@ -416,7 +416,7 @@ def new(cls, config, background_nursery, client_factory=None, with_cre=False, us return bft_network @classmethod - def existing(cls, config, replicas, clients, client_factory=None, background_nursery=None): + def existing(cls, config, replicas, clients, client_factory=None, background_nursery=None, builddir=None): certdir = None builddir = tempfile.mkdtemp(prefix='builddir') if not client_factory: diff --git a/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp b/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp index 686b276ddb..7c68e2094c 100644 --- a/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp +++ b/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp @@ -49,6 +49,8 @@ using Hash = Hasher::Digest; const uint64_t LONG_EXEC_CMD_TIME_IN_SEC = 11; +constexpr char CLIENT_STATE_KEY = 0x1; + template static Hash createHash(const Span &span) { return Hasher{}.digest(span.data(), span.size()); @@ -93,7 +95,7 @@ InternalCommandsHandler::InternalCommandsHandler(concord::kvbc::IReader *storage void InternalCommandsHandler::loadClientStateFromStorage() { ConcordAssert(!ReplicaConfig::instance().isReadOnly); LOG_INFO(GL, "Synchronizing client execution state"); - auto data = m_storage->getLatest(CLIENT_STATE_CAT_ID, {0x1}); + auto data = m_storage->getLatest(CLIENT_STATE_CAT_ID, {CLIENT_STATE_KEY}); if (!data.has_value()) { LOG_WARN(GL, "empty client execution state, were any client requests executed?"); return; @@ -367,7 +369,7 @@ void InternalCommandsHandler::writeAccumulatedBlock(ExecutionRequestsQueue &bloc } VersionedUpdates clientStateUpdate; - clientStateUpdate.addUpdate({0x1}, serializeClientState()); + clientStateUpdate.addUpdate({CLIENT_STATE_KEY}, serializeClientState()); addBlock(verUpdates, merkleUpdates, clientStateUpdate, sn); } @@ -571,7 +573,7 @@ OperationResult InternalCommandsHandler::executeWriteCommand(uint32_t requestSiz VersionedUpdates verUpdates; BlockMerkleUpdates merkleUpdates; VersionedUpdates clientVerUpdates; - clientVerUpdates.addUpdate({0x1}, serializeClientState()); + clientVerUpdates.addUpdate({CLIENT_STATE_KEY}, serializeClientState()); addKeys(write_req, sequenceNum, verUpdates, merkleUpdates); addBlock(verUpdates, merkleUpdates, clientVerUpdates, sequenceNum); } From 020e8775821d36f3137a1522ffd4f5659990de36 Mon Sep 17 00:00:00 2001 From: Ynon Flum Date: Thu, 9 Mar 2023 18:24:41 +0200 Subject: [PATCH 10/18] Implement singleSignatureScheme flag for TesterReplica --- bftengine/src/bftengine/SigManager.cpp | 3 +- .../tests/SigManager/SigManager_test.cpp | 350 ++++++++++++++++++ 2 files changed, 352 insertions(+), 1 deletion(-) diff --git a/bftengine/src/bftengine/SigManager.cpp b/bftengine/src/bftengine/SigManager.cpp index 305bb4b0da..7ba131068e 100644 --- a/bftengine/src/bftengine/SigManager.cpp +++ b/bftengine/src/bftengine/SigManager.cpp @@ -213,10 +213,11 @@ uint16_t SigManager::getSigLength(PrincipalId pid) const { bool SigManager::verifyNonReplicaSig( PrincipalId pid, const concord::Byte* data, size_t dataLength, const concord::Byte* sig, uint16_t sigLength) const { LOG_DEBUG(GL, "Validating NON-replica with: " << KVLOG(myId_, pid, sigLength)); + ConcordAssert(!replicasInfo_.isIdOfReplica(myId_) || !ReplicaConfig::instance().singleSignatureScheme || + !replicasInfo_.isIdOfReplica(pid)); bool result = false; { std::shared_lock lock(mutex_); - ConcordAssert(!replicasInfo_.isIdOfReplica(myId_) || !replicasInfo_.isIdOfReplica(pid)); if (auto pos = verifiers_.find(pid); pos != verifiers_.end()) { result = pos->second->verifyBuffer(data, dataLength, sig, sigLength); diff --git a/bftengine/tests/SigManager/SigManager_test.cpp b/bftengine/tests/SigManager/SigManager_test.cpp index e69de29bb2..1dc04e62eb 100644 --- a/bftengine/tests/SigManager/SigManager_test.cpp +++ b/bftengine/tests/SigManager/SigManager_test.cpp @@ -0,0 +1,350 @@ +// Concord +// +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// +// This product is licensed to you under the Apache 2.0 license (the "License"). You may not use this product except in +// compliance with the Apache 2.0 License. +// +// This product may include a number of subcomponents with separate copyright notices and license terms. Your use of +// these subcomponents is subject to the terms and conditions of the sub-component's license, as noted in the LICENSE +// file. + +#include "SigManager.hpp" +#include "helper.hpp" + +#include + +#include "gtest/gtest.h" +#include "crypto/factory.hpp" +#include "crypto/crypto.hpp" +#include "util/types.hpp" + +using namespace std; +using concord::crypto::KeyFormat; +constexpr size_t RANDOM_DATA_SIZE = 1000U; + +std::default_random_engine generator; + +using concord::crypto::ISigner; +using concord::crypto::IVerifier; +using concord::crypto::Factory; +using bftEngine::ReplicaConfig; +using bftEngine::CryptoManager; +using concord::crypto::SignatureAlgorithm; +using concord::crypto::generateEdDSAKeyPair; + +std::vector> generateKeyPairs(size_t count) { + std::vector> result; + for (size_t i = 0; i < count; i++) { + result.push_back(generateEdDSAKeyPair(KeyFormat::HexaDecimalStrippedFormat)); + } + return result; +} + +void generateRandomData(char* data, size_t len) { + std::uniform_int_distribution distribution(0, 0xFF); + for (size_t i{0}; i < len; ++i) { + data[i] = static_cast(distribution(generator)); + } +} + +void corrupt(concord::Byte* data, size_t len) { + for (size_t i{0}; i < len; ++i) { + data[i] = ~data[i]; + } +} + +void corrupt(char* data, size_t len) { corrupt(reinterpret_cast(data), len); } + +TEST(SignerAndVerifierTest, LoadSignVerifyFromHexKeyPair) { + char data[RANDOM_DATA_SIZE]{0}; + + const auto keyPair = generateEdDSAKeyPair(); + generateRandomData(data, RANDOM_DATA_SIZE); + + const auto signer_ = Factory::getSigner(keyPair.first, ReplicaConfig::instance().replicaMsgSigningAlgo); + const auto verifier_ = Factory::getVerifier(keyPair.second, ReplicaConfig::instance().replicaMsgSigningAlgo); + + // sign with replica signer. + size_t expectedSignerSigLen = signer_->signatureLength(); + std::vector sig(expectedSignerSigLen); + size_t lenRetData; + std::string str_data(data, RANDOM_DATA_SIZE); + lenRetData = signer_->sign(str_data, sig.data()); + ASSERT_EQ(lenRetData, expectedSignerSigLen); + + // validate with replica verifier. + ASSERT_TRUE(verifier_->verify(str_data, sig)); + + // change data randomally, expect failure + char data1[RANDOM_DATA_SIZE]; + std::copy(std::begin(data), std::end(data), std::begin(data1)); + corrupt(data1 + 10, 1); + std::string str_data1(data1, RANDOM_DATA_SIZE); + ASSERT_FALSE(verifier_->verify(str_data1, sig)); + + // change signature randomally, expect failure + corrupt(sig.data(), 1); + str_data = std::string(data, RANDOM_DATA_SIZE); + ASSERT_FALSE(verifier_->verify(str_data, sig)); +} + +TEST(SignerAndVerifierTest, LoadSignVerifyFromPemfiles) { + char data[RANDOM_DATA_SIZE]{0}; + auto keys = generateKeyPairs(1); + auto& [privKey, pubkey] = keys[0]; + + generateRandomData(data, RANDOM_DATA_SIZE); + + const auto signer_ = Factory::getSigner( + privKey, ReplicaConfig::instance().replicaMsgSigningAlgo, KeyFormat::HexaDecimalStrippedFormat); + const auto verifier_ = Factory::getVerifier( + pubkey, ReplicaConfig::instance().replicaMsgSigningAlgo, KeyFormat::HexaDecimalStrippedFormat); + + // sign with replica signer. + size_t expectedSignerSigLen = signer_->signatureLength(); + std::vector sig(expectedSignerSigLen); + std::string str_data(data, RANDOM_DATA_SIZE); + uint32_t lenRetData = signer_->sign(str_data, sig.data()); + ASSERT_EQ(lenRetData, signer_->signatureLength()); + + // validate with replica verifier. + ASSERT_TRUE(verifier_->verify(str_data, sig)); + + // change data randomally, expect failure + char data1[RANDOM_DATA_SIZE]; + std::copy(std::begin(data), std::end(data), std::begin(data1)); + corrupt(data1 + 10, 1); + std::string str_data1(data1, RANDOM_DATA_SIZE); + ASSERT_FALSE(verifier_->verify(str_data1, sig)); + + // change signature randomally, expect failure + corrupt(sig.data(), 1); + str_data = std::string(data, RANDOM_DATA_SIZE); + ASSERT_FALSE(verifier_->verify(str_data, sig)); +} + +class SigManagerTest : public ::testing::Test { + public: + SigManagerTest(bool singleSignatureScheme) + : config{createReplicaConfig()}, + singleSignatureScheme{singleSignatureScheme}, + replicaInfo{config, false, false} {} + + void SetUp() override { + config.singleSignatureScheme = singleSignatureScheme; + + hexKeyPairs = generateKeyPairs(config.numReplicas); + std::set> publicKeysOfReplicas; + std::vector hexReplicaPublicKeys; + std::vector consensusHexReplicaPublicKeys; + + // generateKeyPairs(config.numReplicas + config.numRoReplicas + config.) + for (size_t i = 0; i < config.numReplicas; ++i) { + publicKeysOfReplicas.emplace(i, concord::crypto::EdDSAHexToPem(hexKeyPairs[i]).second); + hexReplicaPublicKeys.push_back(hexKeyPairs[i].second); + } + + sigManager = SigManager::init(config.replicaId, + concord::crypto::EdDSAHexToPem(hexKeyPairs[config.replicaId]).first, + publicKeysOfReplicas, + KeyFormat::PemFormat, + nullptr, + KeyFormat::PemFormat, + {}, + replicaInfo); + + if (singleSignatureScheme) { + consensusHexKeyPairs = hexKeyPairs; + consensusHexReplicaPublicKeys = hexReplicaPublicKeys; + } else { + consensusHexKeyPairs = generateKeyPairs(config.numReplicas); + for (auto& [privateKey, publicKey] : consensusHexKeyPairs) { + consensusHexReplicaPublicKeys.push_back(publicKey); + } + } + + cryptoManager = bftEngine::CryptoManager::init(std::make_unique( + config.replicaId, consensusHexReplicaPublicKeys, consensusHexKeyPairs[config.replicaId].first)); + } + + protected: + std::vector> hexKeyPairs; + std::vector> consensusHexKeyPairs; + std::shared_ptr sigManager; + std::shared_ptr cryptoManager; + ReplicaConfig& config; + const bool singleSignatureScheme; + ReplicasInfo replicaInfo; +}; + +class SingleSchemeSigManagerTest : public SigManagerTest { + public: + SingleSchemeSigManagerTest() : SigManagerTest(true) {} +}; + +class MultiSchemeSigManagerTest : public SigManagerTest { + public: + MultiSchemeSigManagerTest() : SigManagerTest(false) {} +}; + +TEST_F(SingleSchemeSigManagerTest, TestMySigLength) { ASSERT_GT(sigManager->getMySigLength(), 0); } + +TEST_F(SingleSchemeSigManagerTest, ReplicasOnlyCheckVerify) { + const size_t numReplicas = config.numReplicas; + std::vector> signers(numReplicas); + + // Load signers to simulate other replicas + for (size_t i = 0; i < numReplicas; ++i) { + signers[i] = Factory::getSigner( + hexKeyPairs[i].first, ReplicaConfig::instance().replicaMsgSigningAlgo, KeyFormat::HexaDecimalStrippedFormat); + } + + std::vector data(RANDOM_DATA_SIZE); + generateRandomData(data.data(), RANDOM_DATA_SIZE); + + for (size_t i = 0; i < numReplicas; ++i) { + const auto& signer = signers[i]; + + // sign with replica signer (other replicas, mock) + std::vector sig(signer->signatureLength()); + ASSERT_EQ(signer->sign(data, sig.data()), sig.size()); + + // Validate with SigManager (my replica) + ASSERT_EQ(sig.size(), sigManager->getSigLength(i)); + ASSERT_TRUE(sigManager->verifySig(i, data, sig)); + + size_t offset = i % data.size(); + corrupt(data.data() + offset, 1); + ASSERT_FALSE(sigManager->verifySig(i, data, sig)); + } +} + +TEST_F(SingleSchemeSigManagerTest, TestSigIdempotency) { + std::vector msg = {'a', 'b', 'c'}; + const size_t bufferSize = static_cast(sigManager->getMySigLength()); + std::vector firstSig(bufferSize); + std::vector secondSig(bufferSize); + sigManager->sign(0, msg.data(), msg.size(), firstSig.data()); + sigManager->sign(0, msg.data(), msg.size(), secondSig.data()); + ASSERT_EQ(firstSig, secondSig); +} + +TEST_F(MultiSchemeSigManagerTest, TestSigDiffersWithMultipleSchemes) { + std::vector msg = {'a', 'b', 'c'}; + const size_t bufferSize = std::max(static_cast(sigManager->getMySigLength()), + static_cast(cryptoManager->getSigner(0)->signatureLength())); + std::vector firstSig(bufferSize); + std::vector secondSig(bufferSize); + sigManager->sign(0, msg.data(), msg.size(), firstSig.data()); + cryptoManager->getSigner(0)->sign(msg, secondSig.data()); + ASSERT_NE(firstSig, secondSig); +} + +// Check 1 more replica + 1200 clients on 6 additional participants +// where each participant hols a client pool of 200 clients +TEST(SigManagerTestWithClients, ReplicasAndClientsCheckVerify) { + constexpr size_t numReplicas{7}; + constexpr size_t numRoReplicas{2}; + constexpr size_t numOfClientProxies{36}; // (numRoReplicas+numRoReplicas) * 4 + constexpr size_t numParticipantNodes{6}; + constexpr size_t numBftClientsInParticipantNodes{200}; + constexpr size_t totalNumberofExternalBftClients{1200}; // numOfExternaClients * numBftClientsInExternalClient + constexpr PrincipalId myId{0}; + size_t signerIndex{0}; + unique_ptr signers[numReplicas + numParticipantNodes]; // only external clients and consensus replicas sign + + set> publicKeysOfReplicas; + set>> publicKeysOfClients; + unordered_map principalIdToSignerIndex; + + auto keyPairs = generateKeyPairs(numReplicas + numParticipantNodes); + + // Load replica signers to simulate other replicas + PrincipalId currPrincipalId{0}; + for (currPrincipalId = 0; currPrincipalId < numReplicas; ++currPrincipalId) { + signers[signerIndex] = Factory::getSigner(keyPairs[currPrincipalId].first, + ReplicaConfig::instance().replicaMsgSigningAlgo, + KeyFormat::HexaDecimalStrippedFormat); + + publicKeysOfReplicas.insert(make_pair(currPrincipalId, keyPairs[currPrincipalId].second)); + principalIdToSignerIndex.emplace(currPrincipalId, signerIndex); + ++signerIndex; + } + + // Load another group of replica signers to simulate other clients + currPrincipalId = numReplicas + numRoReplicas + numOfClientProxies; + for (size_t i = numReplicas; i < numReplicas + numParticipantNodes; ++i) { + signers[signerIndex] = Factory::getSigner(keyPairs[signerIndex].first, + ReplicaConfig::instance().replicaMsgSigningAlgo, + KeyFormat::HexaDecimalStrippedFormat); + set principalIds; + for (size_t j{0}; j < numBftClientsInParticipantNodes; ++j) { + principalIds.insert(principalIds.end(), currPrincipalId); + principalIdToSignerIndex.insert(make_pair(currPrincipalId, signerIndex)); + ++currPrincipalId; + } + publicKeysOfClients.insert(make_pair(keyPairs[signerIndex].second, std::move(principalIds))); + ++signerIndex; + } + + auto& config = createReplicaConfig(2, 0); + config.numReplicas = numReplicas; + config.numRoReplicas = numRoReplicas; + config.numOfClientProxies = numOfClientProxies; + config.numOfExternalClients = totalNumberofExternalBftClients; + ReplicasInfo replicaInfo(config, false, false); + + shared_ptr sigManager(SigManager::init(myId, + keyPairs[config.replicaId].first, + publicKeysOfReplicas, + KeyFormat::HexaDecimalStrippedFormat, + &publicKeysOfClients, + KeyFormat::HexaDecimalStrippedFormat, + {}, + replicaInfo)); + std::vector replicaHexPublicKeys(config.numReplicas); + for (int j = 0; j < config.numReplicas; j++) { + replicaHexPublicKeys[j] = keyPairs[j].second; + } + CryptoManager::init(std::make_unique( + config.replicaId, replicaHexPublicKeys, keyPairs[config.replicaId].second)); + + // principalIdToSignerIndex carries all principal ids for replica, read only replicas and bft-clients. + // There are some principal Ids in the range [minKey(principalIdToSignerIndex), maxKey(principalIdToSignerIndex)] + // which are not recognized by SigManager - these are the principal ids of Proxy Clients. + // In the next loop we will sign 30K times. Every iteration a random principal ID between minKey and maxKey is + // generated. If the ID is valid, we locate the right signer using principalIdToSignerIndex. If not - sign with + // another signer and expect a failure. + PrincipalId minPidInclusive = 0; + PrincipalId maxPidInclusive = currPrincipalId; + std::uniform_int_distribution distribution(minPidInclusive, maxPidInclusive); + std::vector data(RANDOM_DATA_SIZE); + std::vector sig(1024u); + + for (size_t principalId = 0; principalId < maxPidInclusive; ++principalId) { + size_t lenRetData, signerOffset; + + auto iter = principalIdToSignerIndex.find(principalId); + if (iter == principalIdToSignerIndex.end()) { + ASSERT_FALSE(sigManager->hasVerifier(principalId)); + ASSERT_EQ(sigManager->getSigLength(principalId), 0); + continue; + } + signerOffset = iter->second; + + // sign + LOG_DEBUG(GL, KVLOG(principalId, signerOffset, config.replicaId)); + auto expectedSignerSigLen = signers[signerOffset]->signatureLength(); + ASSERT_GE(sig.size(), expectedSignerSigLen); + generateRandomData(data.data(), RANDOM_DATA_SIZE); + lenRetData = signers[signerOffset]->sign(data, sig.data()); + ASSERT_EQ(lenRetData, expectedSignerSigLen); + + // Validate with SigManager (my replica) + ASSERT_TRUE(sigManager->getSigLength(principalId) > 0); + bool signatureValid = sigManager->verifySig( + principalId, data, string_view{reinterpret_cast(sig.data()), sigManager->getSigLength(principalId)}); + ASSERT_TRUE(signatureValid); + } +} From 97da281302515069953e0b5fa036c6f3142a503d Mon Sep 17 00:00:00 2001 From: Ynon Flum Date: Thu, 16 Mar 2023 10:42:29 +0200 Subject: [PATCH 11/18] Refine SigManager-CryptoManager interface --- bftengine/include/bftengine/CryptoManager.hpp | 9 +-- .../bcstatetransfer/AsyncStateTransferCRE.cpp | 6 +- bftengine/src/bftengine/CryptoManager.cpp | 30 +++------ bftengine/src/bftengine/SigManager.cpp | 65 ++++++++++++------- bftengine/src/bftengine/SigManager.hpp | 13 ++-- .../tests/SigManager/SigManager_test.cpp | 15 ++++- 6 files changed, 77 insertions(+), 61 deletions(-) diff --git a/bftengine/include/bftengine/CryptoManager.hpp b/bftengine/include/bftengine/CryptoManager.hpp index f7d1c01a24..cab8f198a6 100644 --- a/bftengine/include/bftengine/CryptoManager.hpp +++ b/bftengine/include/bftengine/CryptoManager.hpp @@ -18,8 +18,6 @@ #include "crypto/threshsign/ThresholdSignaturesTypes.h" #include "crypto/threshsign/IThresholdSigner.h" #include "crypto/threshsign/IThresholdVerifier.h" -#include "crypto/threshsign/eddsa/EdDSAMultisigVerifier.h" -#include "crypto/threshsign/eddsa/EdDSAMultisigSigner.h" #include "ReplicaConfig.hpp" #include "IKeyExchanger.hpp" #include "crypto/crypto.hpp" @@ -91,10 +89,9 @@ class CryptoManager : public IKeyExchanger, public IMultiSigKeyGenerator { // This abstraction is broken to allow using the consensus key as the replica's main key (In SigManager), thus // enabling an operator to change it (key rotation). // This code will need to be refactored if a different implementation is used. - std::shared_ptr getSigner(SeqNum seq) const; - std::shared_ptr getMultisigVerifier(SeqNum seq) const; - std::array>, 2> getLatestVerifiers() const; - std::array, 2> getLatestSigners() const; + std::shared_ptr getSigner(SeqNum seq) const; + std::array>, 2> getLatestVerifiers() const; + std::array, 2> getLatestSigners() const; private: /** diff --git a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp index 276dfe8a3c..94875a0d79 100644 --- a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp +++ b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp @@ -199,9 +199,7 @@ size_t ReplicaCRESigner::signBuffer(const concord::Byte* dataIn, size_t dataLen, return sigManager->sign(sigManager->getReplicaLastExecutedSeq(), dataIn, dataLen, sigOutBuffer); } -size_t ReplicaCRESigner::signatureLength() const { - return bftEngine::impl::SigManager::instance()->getLastReplicaSigner()->signatureLength(); -} -std::string ReplicaCRESigner::getPrivKey() const { ConcordAssert(false); } +size_t ReplicaCRESigner::signatureLength() const { return bftEngine::impl::SigManager::instance()->getMySigLength(); } +std::string ReplicaCRESigner::getPrivKey() const { return bftEngine::impl::SigManager::instance()->getSelfPrivKey(); } } // namespace bftEngine::bcst::asyncCRE diff --git a/bftengine/src/bftengine/CryptoManager.cpp b/bftengine/src/bftengine/CryptoManager.cpp index c32d6f086b..6c9cafc57b 100644 --- a/bftengine/src/bftengine/CryptoManager.cpp +++ b/bftengine/src/bftengine/CryptoManager.cpp @@ -116,40 +116,28 @@ void CryptoManager::onCheckpoint(uint64_t newCheckpoint) { assertMapSizeValid(); } -std::shared_ptr CryptoManager::getSigner(SeqNum seq) const { - auto signer = get(seq)->thresholdSigner_; - return std::reinterpret_pointer_cast(signer); -} - -std::shared_ptr CryptoManager::getMultisigVerifier(SeqNum seq) const { - auto verifier = get(seq)->thresholdVerifierForOptimisticCommit_; - return std::reinterpret_pointer_cast(verifier); -} +std::shared_ptr CryptoManager::getSigner(SeqNum seq) const { return get(seq)->thresholdSigner_; } -std::array>, 2> CryptoManager::getLatestVerifiers() const { +std::array>, 2> CryptoManager::getLatestVerifiers() const { std::lock_guard guard(mutex_); auto riter = checkpointToSystem().rbegin(); - std::array>, 2> result; - result[0] = { - riter->first, - std::reinterpret_pointer_cast(riter->second->thresholdVerifierForOptimisticCommit_)}; + std::array>, 2> result; + result[0] = {riter->first, riter->second->thresholdVerifierForOptimisticCommit_}; riter++; if (riter != checkpointToSystem().rend() && riter->second != nullptr) { - result[1] = { - riter->first, - std::reinterpret_pointer_cast(riter->second->thresholdVerifierForOptimisticCommit_)}; + result[1] = {riter->first, riter->second->thresholdVerifierForOptimisticCommit_}; } return result; } -std::array, 2> CryptoManager::getLatestSigners() const { +std::array, 2> CryptoManager::getLatestSigners() const { std::lock_guard guard(mutex_); auto riter = checkpointToSystem().rbegin(); - std::array, 2> result; - result[0] = std::reinterpret_pointer_cast(riter->second->thresholdSigner_); + std::array, 2> result; + result[0] = riter->second->thresholdSigner_; ++riter; if (riter != checkpointToSystem().rend() && riter->second != nullptr) { - result[1] = std::reinterpret_pointer_cast(riter->second->thresholdSigner_); + result[1] = riter->second->thresholdSigner_; } return result; } diff --git a/bftengine/src/bftengine/SigManager.cpp b/bftengine/src/bftengine/SigManager.cpp index 7ba131068e..705d55a85c 100644 --- a/bftengine/src/bftengine/SigManager.cpp +++ b/bftengine/src/bftengine/SigManager.cpp @@ -261,9 +261,9 @@ bool SigManager::verifyNonReplicaSig( size_t SigManager::sign(SeqNum seq, const concord::Byte* data, size_t dataLength, concord::Byte* outSig) const { ConcordAssert(replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica()); concord::crypto::ISigner* rawSigner = nullptr; - std::shared_ptr signer; + std::shared_ptr signer; if (ReplicaConfig::instance().singleSignatureScheme) { - signer = CryptoManager::instance().getSigner(seq); + signer = extractSignerFromMultisig(CryptoManager::instance().getSigner(seq)); rawSigner = signer.get(); } else { rawSigner = mySigner_.get(); @@ -285,27 +285,29 @@ bool SigManager::verifyReplicaSig(PrincipalId replicaID, LOG_DEBUG(GL, KVLOG(myId_)); ConcordAssert(replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica()); ConcordAssert(replicasInfo_.isIdOfReplica(replicaID)); - int i = 0; + int lastVerifierIndex = 0; for (auto [seq, multisigVerifier] : CryptoManager::instance().getLatestVerifiers()) { UNUSED(seq); if (multisigVerifier.get() == nullptr) { continue; } - auto& verifier = multisigVerifier->getVerifier(replicaID); + auto& verifier = extractVerifierFromMultisig(multisigVerifier, replicaID); LOG_DEBUG(GL, "Validating as replica with: " << KVLOG( - myId_, replicaID, i, verifier.getPubKey(), sigLength, verifier.signatureLength())); + myId_, replicaID, lastVerifierIndex, verifier.getPubKey(), sigLength, verifier.signatureLength())); if (verifier.verifyBuffer(data, dataLength, sig, sigLength)) { - LOG_DEBUG(GL, "Validation Successful " << KVLOG(myId_, replicaID, i, sigLength, verifier.signatureLength())); + LOG_DEBUG(GL, + "Validation Successful " << KVLOG( + myId_, replicaID, lastVerifierIndex, sigLength, verifier.signatureLength())); return true; } else { - LOG_DEBUG( - GL, - "Validation failed as replica with: " << KVLOG(myId_, replicaID, i, sigLength, verifier.signatureLength())); + LOG_DEBUG(GL, + "Validation failed as replica with: " << KVLOG( + myId_, replicaID, lastVerifierIndex, sigLength, verifier.signatureLength())); } - i++; + lastVerifierIndex++; } - LOG_WARN(GL, "Validation failed with all cryptosystems" << KVLOG(myId_, replicaID, i)); + LOG_WARN(GL, "Validation failed with all cryptosystems" << KVLOG(myId_, replicaID, lastVerifierIndex)); return false; } @@ -326,7 +328,7 @@ bool SigManager::verifyOwnSignature(const concord::Byte* data, if (ReplicaConfig::instance().singleSignatureScheme) { auto signers = CryptoManager::instance().getLatestSigners(); for (auto& signer : signers) { - signer->signBuffer(data, dataLength, sig.data()); + extractSignerFromMultisig(signer)->signBuffer(data, dataLength, sig.data()); if (std::memcmp(sig.data(), expectedSignature, getMySigLength()) == 0) { LOG_DEBUG(GL, "Self-sig validation succeeded"); @@ -342,10 +344,32 @@ bool SigManager::verifyOwnSignature(const concord::Byte* data, return std::memcmp(sig.data(), expectedSignature, getMySigLength()) == 0; } +std::shared_ptr SigManager::extractSignerFromMultisig( + std::shared_ptr thresholdSigner) const { + ConcordAssert(ReplicaConfig::instance().singleSignatureScheme); + ConcordAssertNE(thresholdSigner.get(), nullptr); + std::shared_ptr eddsaSigner = std::dynamic_pointer_cast(thresholdSigner); + ConcordAssertNE(eddsaSigner.get(), nullptr); + std::shared_ptr baseSigner = + std::dynamic_pointer_cast(eddsaSigner); + ConcordAssertNE(baseSigner.get(), nullptr); + return baseSigner; +} + +const concord::crypto::IVerifier& SigManager::extractVerifierFromMultisig( + std::shared_ptr thresholdVerifier, PrincipalId id) const { + ConcordAssert(ReplicaConfig::instance().singleSignatureScheme); + ConcordAssertNE(thresholdVerifier.get(), nullptr); + std::shared_ptr eddsaVerifier = + std::dynamic_pointer_cast(thresholdVerifier); + ConcordAssertNE(eddsaVerifier.get(), nullptr); + return eddsaVerifier->getVerifier(id); +} + uint16_t SigManager::getMySigLength() const { if (ReplicaConfig::instance().singleSignatureScheme && (replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica())) { - return getCurrentReplicaSigner()->signatureLength(); + return extractSignerFromMultisig(getCurrentReplicaSigner())->signatureLength(); } return (uint16_t)mySigner_->signatureLength(); } @@ -371,20 +395,13 @@ bool SigManager::hasVerifier(PrincipalId pid) { return verifiers_.find(pid) != v concord::crypto::SignatureAlgorithm SigManager::getMainKeyAlgorithm() const { return concord::crypto::EdDSA; } -std::shared_ptr SigManager::getCurrentReplicaSigner() const { +std::shared_ptr SigManager::getCurrentReplicaSigner() const { ConcordAssert(ReplicaConfig::instance().singleSignatureScheme); ConcordAssert(replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica()); auto signer = CryptoManager::instance().getSigner(getReplicaLastExecutedSeq()); return signer; } -std::shared_ptr SigManager::getLastReplicaSigner() const { - ConcordAssert(ReplicaConfig::instance().singleSignatureScheme); - ConcordAssert(replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica()); - auto latestSigner = CryptoManager::instance().getLatestSigners()[0]; - return latestSigner; -} - std::pair SigManager::getMyLatestPublicKey() const { ConcordAssert(replicasInfo_.isIdOfReplica(myId_)); @@ -393,14 +410,14 @@ std::pair SigManager::getMyLatestPublicKey() const { } auto [seq, latestVerifier] = CryptoManager::instance().getLatestVerifiers()[0]; - auto pubKey = latestVerifier->getVerifier(myId_).getPubKey(); + auto pubKey = extractVerifierFromMultisig(latestVerifier, myId_).getPubKey(); LOG_DEBUG(GL, KVLOG(seq, pubKey)); return {seq, pubKey}; } std::string SigManager::getSelfPrivKey() const { if (ReplicaConfig::instance().singleSignatureScheme) { - return getCurrentReplicaSigner()->getPrivKey(); + return extractSignerFromMultisig(getCurrentReplicaSigner())->getPrivKey(); } return mySigner_->getPrivKey(); } @@ -415,7 +432,7 @@ std::array SigManager::getPublicKeyOfV if (multisigVerifier.get() == nullptr) { continue; } - auto& verifier = multisigVerifier->getVerifier(id); + auto& verifier = extractVerifierFromMultisig(multisigVerifier, id); publicKeys[i] = verifier.getPubKey(); ++i; } diff --git a/bftengine/src/bftengine/SigManager.hpp b/bftengine/src/bftengine/SigManager.hpp index 9fd5c2044b..095bbce42e 100644 --- a/bftengine/src/bftengine/SigManager.hpp +++ b/bftengine/src/bftengine/SigManager.hpp @@ -29,8 +29,8 @@ using concordMetrics::AtomicCounterHandle; -class EdDSAMultisigSigner; -class EdDSAMultisigVerifier; +class IThresholdSigner; +class IThresholdVerifier; namespace bftEngine { namespace impl { @@ -105,8 +105,6 @@ class SigManager { SigManager& operator=(SigManager&&) = delete; concord::crypto::SignatureAlgorithm getMainKeyAlgorithm() const; - std::shared_ptr getCurrentReplicaSigner() const; - std::shared_ptr getLastReplicaSigner() const; const concord::crypto::IVerifier& getVerifier(PrincipalId otherPrincipal) const; std::string getClientsPublicKeys(); @@ -126,6 +124,11 @@ class SigManager { bool verifyOwnSignature(const concord::Byte* data, size_t dataLength, const concord::Byte* expectedSignature) const; + std::shared_ptr extractSignerFromMultisig( + std::shared_ptr thresholdSigner) const; + const concord::crypto::IVerifier& extractVerifierFromMultisig(std::shared_ptr thresholdVerifier, + PrincipalId id) const; + protected: static constexpr uint16_t updateMetricsAggregatorThresh = 1000; @@ -143,6 +146,8 @@ class SigManager { const concord::Byte* sig, uint16_t sigLength) const; + std::shared_ptr getCurrentReplicaSigner() const; + const PrincipalId myId_; SeqNum replicaLastExecutedSeq_{0}; std::unique_ptr mySigner_; diff --git a/bftengine/tests/SigManager/SigManager_test.cpp b/bftengine/tests/SigManager/SigManager_test.cpp index 1dc04e62eb..40f6cbe6fd 100644 --- a/bftengine/tests/SigManager/SigManager_test.cpp +++ b/bftengine/tests/SigManager/SigManager_test.cpp @@ -17,6 +17,7 @@ #include "gtest/gtest.h" #include "crypto/factory.hpp" #include "crypto/crypto.hpp" +#include "crypto/threshsign/eddsa/EdDSAMultisigSigner.h" #include "util/types.hpp" using namespace std; @@ -188,6 +189,15 @@ class MultiSchemeSigManagerTest : public SigManagerTest { MultiSchemeSigManagerTest() : SigManagerTest(false) {} }; +TEST_F(SingleSchemeSigManagerTest, TestSignerExtraction) { + ASSERT_EQ(sigManager->extractSignerFromMultisig(cryptoManager->getSigner(0))->getPrivKey(), hexKeyPairs[0].first); +} + +TEST_F(SingleSchemeSigManagerTest, TestVerifierExtraction) { + ASSERT_EQ(sigManager->extractVerifierFromMultisig(cryptoManager->getLatestVerifiers()[0].second, 0).getPubKey(), + hexKeyPairs[0].second); +} + TEST_F(SingleSchemeSigManagerTest, TestMySigLength) { ASSERT_GT(sigManager->getMySigLength(), 0); } TEST_F(SingleSchemeSigManagerTest, ReplicasOnlyCheckVerify) { @@ -232,12 +242,13 @@ TEST_F(SingleSchemeSigManagerTest, TestSigIdempotency) { TEST_F(MultiSchemeSigManagerTest, TestSigDiffersWithMultipleSchemes) { std::vector msg = {'a', 'b', 'c'}; + EdDSAMultisigSigner* multisigSigner = reinterpret_cast(cryptoManager->getSigner(0).get()); const size_t bufferSize = std::max(static_cast(sigManager->getMySigLength()), - static_cast(cryptoManager->getSigner(0)->signatureLength())); + static_cast(multisigSigner->signatureLength())); std::vector firstSig(bufferSize); std::vector secondSig(bufferSize); sigManager->sign(0, msg.data(), msg.size(), firstSig.data()); - cryptoManager->getSigner(0)->sign(msg, secondSig.data()); + multisigSigner->sign(msg, secondSig.data()); ASSERT_NE(firstSig, secondSig); } From 15adec0a1bf673c0a48d8cb4adf0f435f011ad18 Mon Sep 17 00:00:00 2001 From: Ynon Flum Date: Sun, 19 Mar 2023 11:40:13 +0200 Subject: [PATCH 12/18] Fix documentation, rename functions, re-enable stubbed apollo tests --- bftengine/include/bftengine/CryptoManager.hpp | 2 +- .../include/bftengine/KeyExchangeManager.hpp | 5 ++-- .../bcstatetransfer/AsyncStateTransferCRE.cpp | 11 ++++---- .../bcstatetransfer/AsyncStateTransferCRE.hpp | 16 +++++------ .../CollectorOfThresholdSignatures.hpp | 2 -- bftengine/src/bftengine/CryptoManager.cpp | 4 +-- .../src/bftengine/KeyExchangeManager.cpp | 16 +++++------ bftengine/src/bftengine/ReplicaBase.hpp | 4 ++- bftengine/src/bftengine/ReplicaImp.cpp | 28 +++++++++++-------- bftengine/src/bftengine/ReplicasInfo.cpp | 2 +- bftengine/src/bftengine/SigManager.cpp | 4 ++- bftengine/src/bftengine/SigManager.hpp | 2 +- .../src/bftengine/messages/ViewChangeMsg.cpp | 2 -- .../poll_based_state_client.hpp | 8 ++++++ tests/apollo/test_skvbc_reconfiguration.py | 2 +- tests/apollo/test_skvbc_state_transfer.py | 3 +- tests/apollo/test_skvbc_view_change.py | 2 ++ tools/KeyfileIOUtils.cpp | 17 +++++------ 18 files changed, 73 insertions(+), 57 deletions(-) diff --git a/bftengine/include/bftengine/CryptoManager.hpp b/bftengine/include/bftengine/CryptoManager.hpp index cab8f198a6..e26981d1cc 100644 --- a/bftengine/include/bftengine/CryptoManager.hpp +++ b/bftengine/include/bftengine/CryptoManager.hpp @@ -1,6 +1,6 @@ // Concord // -// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. // // This product is licensed to you under the Apache 2.0 license (the "License"). You may not use this product except in // compliance with the Apache 2.0 License. diff --git a/bftengine/include/bftengine/KeyExchangeManager.hpp b/bftengine/include/bftengine/KeyExchangeManager.hpp index acba50c06d..d563f6d1ab 100644 --- a/bftengine/include/bftengine/KeyExchangeManager.hpp +++ b/bftengine/include/bftengine/KeyExchangeManager.hpp @@ -57,9 +57,10 @@ class KeyExchangeManager { // Called at the end of state transfer void loadPublicKeys(); void loadClientPublicKeys(); - // whether initial key exchange has occurred - bool exchanged() const; + // True if all of the replicas have completed a key rotation on startup. + // The startup key rotation is only executed if ReplicaConfig::instance().getkeyExchangeOnStart() is true. + bool isInitialConsensusExchangeComplete() const; const std::string kInitialKeyExchangeCid = "KEY-EXCHANGE-"; const std::string kInitialClientsKeysCid = "CLIENTS-PUB-KEYS-"; ///////// Clients public keys interface/////////////// diff --git a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp index 94875a0d79..a07ffda586 100644 --- a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp +++ b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp @@ -176,15 +176,16 @@ std::shared_ptr CreFactory::create( } bftClientConf.replicas_master_key_folder_path = std::nullopt; std::unique_ptr comm = std::make_unique(msgsCommunicator, msgHandlers); - bft::client::Client* bftClient = new bft::client::Client(std::move(comm), bftClientConf); + auto bftClient = std::make_unique(std::move(comm), bftClientConf); bftClient->setTransactionSigner(transactionSigner.release()); Config cre_config; cre_config.id_ = repConfig.replicaId; cre_config.interval_timeout_ms_ = 1000; - // TODO: fix relying on f + 1, so that byzantine replicas are also handled - IStateClient* pbc = new PollBasedStateClient(bftClient, cre_config.interval_timeout_ms_, 0, cre_config.id_, true); - auto cre = - std::make_shared(cre_config, pbc, std::make_shared()); + // TODO: Fix relying on f + 1, so that byzantine replicas are also handled + auto pbc = std::unique_ptr( + new PollBasedStateClient(bftClient.release(), cre_config.interval_timeout_ms_, 0, cre_config.id_, true)); + auto cre = std::make_shared( + cre_config, pbc.release(), std::make_shared()); if (bftEngine::ReplicaConfig::instance().isReadOnly) { cre->registerHandler(std::make_shared()); } else { diff --git a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.hpp b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.hpp index aaf5e06749..78bedac00d 100644 --- a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.hpp +++ b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.hpp @@ -23,20 +23,20 @@ namespace bftEngine::bcst::asyncCRE { class CreFactory { public: + /* + * TODO: Transaction signing needs to be disabled for replicas, as state transfer may change + * a replica's consensus (and therefore main) key. Apollo tests currently run with transaction signing enabled, + * we therefore use the latest key published by this replica. + * If a consensus was reached over a key exchange which was initiated by this replica + * prior to issuing CRE client requests, the other replicas are guaranteed to be able to use + * this replica's latest main key assuming an honest execution of the exchange. + */ static std::shared_ptr create( std::shared_ptr msgsCommunicator, std::shared_ptr msgHandlers, std::unique_ptr transactionSigner); }; -/* - * TODO: Transaction signing needs to be disabled for replicas, as state transfer may change - * a replica's consensus (and therefore main) key. Apollo tests currently run with transaction signing enabled, - * we therefore use the latest key published by this replica. - * If a consensus was reached over a key exchange which was initiated by this replica - * prior to issuing CRE client requests, the other replicas are guaranteed to be able to use - * this replica's latest main key assuming an honest execution of the exchange. - */ class ReplicaCRESigner : public concord::crypto::ISigner { public: size_t signBuffer(const concord::Byte* dataIn, size_t dataLen, concord::Byte* sigOutBuffer) const override; diff --git a/bftengine/src/bftengine/CollectorOfThresholdSignatures.hpp b/bftengine/src/bftengine/CollectorOfThresholdSignatures.hpp index e95dfe8424..5409edda92 100644 --- a/bftengine/src/bftengine/CollectorOfThresholdSignatures.hpp +++ b/bftengine/src/bftengine/CollectorOfThresholdSignatures.hpp @@ -393,8 +393,6 @@ class CollectorOfThresholdSignatures { fullSignedDataLength = acc->getFullSignedData(bufferForSigComputations.data(), bufferSize); if (!verifier->verify( (char*)&expectedDigest, sizeof(Digest), bufferForSigComputations.data(), fullSignedDataLength)) { - // if verification failed again - // signer index starts with 1, therefore shareId-1 std::set replicasWithBadSigs; for (const auto& shareId : acc->getInvalidShareIds()) replicasWithBadSigs.insert((uint16_t)shareId); // send failed message diff --git a/bftengine/src/bftengine/CryptoManager.cpp b/bftengine/src/bftengine/CryptoManager.cpp index 6c9cafc57b..ffe7ff6c75 100644 --- a/bftengine/src/bftengine/CryptoManager.cpp +++ b/bftengine/src/bftengine/CryptoManager.cpp @@ -133,6 +133,7 @@ std::array>, 2> CryptoMana std::array, 2> CryptoManager::getLatestSigners() const { std::lock_guard guard(mutex_); auto riter = checkpointToSystem().rbegin(); + ConcordAssertNE(riter->second, nullptr); std::array, 2> result; result[0] = riter->second->thresholdSigner_; ++riter; @@ -172,13 +173,12 @@ CheckpointNum CryptoManager::getCheckpointOfCryptosystemForSeq(const SeqNum sn) } } - // Replicas might encounter old messages, crashing is not + // Replicas might encounter old messages, crashing is not a valid behavior auto fallbackSystemCheckpoint = checkpointToSystem().rbegin()->first; LOG_WARN(logger(), "Cryptosystem not found, returning latest" << KVLOG(sn, checkpointUpperBound, checkpointToSystem().size(), fallbackSystemCheckpoint)); return fallbackSystemCheckpoint; - // ConcordAssert(false && "should never reach here"); } std::shared_ptr CryptoManager::get(const SeqNum& sn) const { diff --git a/bftengine/src/bftengine/KeyExchangeManager.cpp b/bftengine/src/bftengine/KeyExchangeManager.cpp index 3c4448ca29..3163cccb2d 100644 --- a/bftengine/src/bftengine/KeyExchangeManager.cpp +++ b/bftengine/src/bftengine/KeyExchangeManager.cpp @@ -104,11 +104,11 @@ std::string KeyExchangeManager::onKeyExchange(const KeyExchangeMsg& kemsg, // The key will be used from this sequence forwards // new keys are used two checkpoints after reaching consensus const SeqNum sn = kemsg.generated_sn; - LOG_INFO(KEY_EX_LOG, KVLOG(kemsg.toString(), sn, cid, exchanged())); + LOG_INFO(KEY_EX_LOG, KVLOG(kemsg.toString(), sn, cid, isInitialConsensusExchangeComplete())); // client query if (kemsg.op == KeyExchangeMsg::HAS_KEYS) { - LOG_INFO(KEY_EX_LOG, "Has keys: " << std::boolalpha << exchanged() << std::noboolalpha); - if (!exchanged()) return std::string(KeyExchangeMsg::hasKeysFalseReply); + LOG_INFO(KEY_EX_LOG, "Has keys: " << std::boolalpha << isInitialConsensusExchangeComplete() << std::noboolalpha); + if (!isInitialConsensusExchangeComplete()) return std::string(KeyExchangeMsg::hasKeysFalseReply); return std::string(KeyExchangeMsg::hasKeysTrueReply); } uint64_t my_epoch = EpochManager::instance().getSelfEpochNumber(); @@ -129,8 +129,8 @@ std::string KeyExchangeManager::onKeyExchange(const KeyExchangeMsg& kemsg, if (ReplicaConfig::instance().getkeyExchangeOnStart() && (publicKeys_.numOfExchangedReplicas() <= liveQuorumSize)) LOG_INFO(KEY_EX_LOG, "Exchanged [" << publicKeys_.numOfExchangedReplicas() << "] out of [" << liveQuorumSize << "]" - << KVLOG(kemsg.repID, initial_exchange_, exchanged())); - if (!initial_exchange_ && exchanged()) { + << KVLOG(kemsg.repID, initial_exchange_, isInitialConsensusExchangeComplete())); + if (!initial_exchange_ && isInitialConsensusExchangeComplete()) { initial_exchange_ = true; if (ReplicaConfig::instance().getkeyExchangeOnStart() && ReplicaConfig::instance().publishReplicasMasterKeyOnStartup) { @@ -170,7 +170,7 @@ void KeyExchangeManager::loadPublicKeys() { // after State Transfer public keys for all replicas are expected to exist auto num_loaded = publicKeys_.loadAllReplicasKeyStoresFromReservedPages(); uint32_t liveQuorumSize = ReplicaConfig::instance().waitForFullCommOnStartup ? clusterSize_ : quorumSize_; - if (ReplicaConfig::instance().getkeyExchangeOnStart() && exchanged()) { + if (ReplicaConfig::instance().getkeyExchangeOnStart() && isInitialConsensusExchangeComplete()) { ConcordAssertGE(num_loaded, liveQuorumSize); } LOG_INFO(KEY_EX_LOG, "rebuilding crypto system after state transfer" << KVLOG(num_loaded)); @@ -440,7 +440,7 @@ std::string KeyExchangeManager::getStatus() const { using concordUtils::toPair; std::ostringstream oss; std::unordered_map result; - result.insert(toPair("exchanged", exchanged())); + result.insert(toPair("exchanged", isInitialConsensusExchangeComplete())); result.insert(toPair("sent_key_exchange_on_start", metrics_->sent_key_exchange_on_start_status.Get().Get())); result.insert(toPair("sent_key_exchange", metrics_->sent_key_exchange_counter.Get().Get())); result.insert(toPair("self_key_exchange", metrics_->self_key_exchange_counter.Get().Get())); @@ -469,7 +469,7 @@ bool KeyExchangeManager::PrivateKeys::load() { return true; } -bool KeyExchangeManager::exchanged() const { +bool KeyExchangeManager::isInitialConsensusExchangeComplete() const { uint32_t liveClusterSize = ReplicaConfig::instance().waitForFullCommOnStartup ? clusterSize_ : quorumSize_; bool exchange_self_keys = publicKeys_.keyExists(ReplicaConfig::instance().replicaId); LOG_DEBUG(KEY_EX_LOG, diff --git a/bftengine/src/bftengine/ReplicaBase.hpp b/bftengine/src/bftengine/ReplicaBase.hpp index 4385d1d163..aa625a4ea3 100644 --- a/bftengine/src/bftengine/ReplicaBase.hpp +++ b/bftengine/src/bftengine/ReplicaBase.hpp @@ -88,11 +88,13 @@ class ReplicaBase { } try { /* Checkpoint messages are used by replicas to initiate state transfer. - * Since a replica which initiates state transfer might not have the most + * Since a replica which initiates state transfer might not have the most recent * public keys of other replicas, it might not be able to validate the signature * of checkpoint messages. State transfer only requires that a replica has a valid * communication channel. * Therefore, replicas don't validate signatures of checkpoint messages. + * Note: When using UDP communication channels, a byzantine replica can cause + * other replicas to initiate state transfer. */ if constexpr (std::is_same_v) { msg->validate(*repsInfo, false); diff --git a/bftengine/src/bftengine/ReplicaImp.cpp b/bftengine/src/bftengine/ReplicaImp.cpp index 990c4dd0f9..1ecd2fd8cb 100644 --- a/bftengine/src/bftengine/ReplicaImp.cpp +++ b/bftengine/src/bftengine/ReplicaImp.cpp @@ -396,13 +396,14 @@ void ReplicaImp::onMessage(std::unique_ptr m // - in case replica keys haven't been exchanged for all replicas, and it's not a key exchange msg then don't accept // the msgs. // - the public keys of the clients haven't been published yet. - if (!KeyExchangeManager::instance().exchanged() || + if (!KeyExchangeManager::instance().isInitialConsensusExchangeComplete() || (!KeyExchangeManager::instance().clientKeysPublished() && repsInfo->isIdOfClientProxy(senderId))) { if (!(flags & KEY_EXCHANGE_FLAG) && !(flags & CLIENTS_PUB_KEYS_FLAG)) { LOG_INFO(KEY_EX_LOG, - "Didn't complete yet, dropping msg" << KVLOG(KeyExchangeManager::instance().exchanged(), - KeyExchangeManager::instance().clientKeysPublished(), - repsInfo->isIdOfClientProxy(senderId))); + "Didn't complete yet, dropping msg" + << KVLOG(KeyExchangeManager::instance().isInitialConsensusExchangeComplete(), + KeyExchangeManager::instance().clientKeysPublished(), + repsInfo->isIdOfClientProxy(senderId))); return; } } @@ -2250,6 +2251,11 @@ void ReplicaImp::onFastPathCommitVerifyCombinedSigResult(SeqNum seqNumber, startExecution(seqNumber, span, askForMissingInfoAboutCommittedItems); } +/** + * Replicas accumulate checkpoints messages to decide whether they should start + * state transfer. Checkpoint message signatures are not validated, as replicas + * rely on state transfer to fetch the most recent keys. + */ template <> void ReplicaImp::onMessage(std::unique_ptr message) { auto *msg = message.release(); @@ -2801,10 +2807,6 @@ void ReplicaImp::onMessage(std::unique_ptr m } } - /*if (msgLastStable > lastStableSeqNum + (checkpointWindowSize * 2) && !isCollectingState()) { - startCollectingState("On receiving ReplicaStatusMsg"); - }*/ - delete msg; } @@ -4405,7 +4407,7 @@ ReplicaImp::ReplicaImp(bool firstTime, LOG_INFO(GL, "Initialising Replica" << KVLOG(firstTime)); onViewNumCallbacks_.add([&](bool) { - if (config_.keyExchangeOnStart && !KeyExchangeManager::instance().exchanged()) { + if (config_.keyExchangeOnStart && !KeyExchangeManager::instance().isInitialConsensusExchangeComplete()) { LOG_INFO(GL, "key exchange has not been finished yet. Give it another try"); KeyExchangeManager::instance().waitForQuorumAndTriggerConsensusExchange(this); } @@ -4415,7 +4417,8 @@ ReplicaImp::ReplicaImp(bool firstTime, // which syncs its state through ST, we need to make sure that it completes // initial key exchange after completing ST if (!isCollectingState()) { - if (ReplicaConfig::instance().getkeyExchangeOnStart() && !KeyExchangeManager::instance().exchanged()) { + if (ReplicaConfig::instance().getkeyExchangeOnStart() && + !KeyExchangeManager::instance().isInitialConsensusExchangeComplete()) { KeyExchangeManager::instance().waitForQuorumAndTriggerConsensusExchange(this, lastExecutedSeqNum); LOG_INFO(GL, "Send key exchange after completing state transfer " << KVLOG(lastExecutedSeqNum)); } @@ -4650,7 +4653,8 @@ void ReplicaImp::start() { // It must happen after the replica recovers requests in the main thread. msgsCommunicator_->startMsgsProcessing(config_.getreplicaId()); - if (ReplicaConfig::instance().getkeyExchangeOnStart() && !KeyExchangeManager::instance().exchanged()) { + if (ReplicaConfig::instance().getkeyExchangeOnStart() && + !KeyExchangeManager::instance().isInitialConsensusExchangeComplete()) { KeyExchangeManager::instance().waitForQuorumAndTriggerConsensusExchange(this); } else { // If key exchange is disabled, first publish the replica's main key to clients @@ -5695,7 +5699,7 @@ void ReplicaImp::sendResponses(PrePrepareMsg *ppMsg, IRequestsHandler::Execution req.outReplicaSpecificInfoSize, executionResult); if (replyMsg) { - LOG_INFO(GL, "Sending reply for req " << req.requestSequenceNum); + LOG_DEBUG(GL, "Sending reply for req " << req.requestSequenceNum); send(replyMsg.get(), req.clientId); free(req.outReply); } else { diff --git a/bftengine/src/bftengine/ReplicasInfo.cpp b/bftengine/src/bftengine/ReplicasInfo.cpp index 3d1bb7dc75..13d3060b57 100644 --- a/bftengine/src/bftengine/ReplicasInfo.cpp +++ b/bftengine/src/bftengine/ReplicasInfo.cpp @@ -55,7 +55,7 @@ ReplicasInfo::ReplicasInfo(const ReplicaConfig& config, bool dynamicCollectorForPartialProofs, bool dynamicCollectorForExecutionProofs) : _myId{config.replicaId}, - _isRoReplica{(_myId >= config.numReplicas) && (_myId < config.numReplicas + config.numRoReplicas)}, + _isRoReplica{config.isReadOnly}, _numberOfReplicas{config.numReplicas}, _numberOfRoReplicas{config.numRoReplicas}, _numOfClientProxies{config.numOfClientProxies}, diff --git a/bftengine/src/bftengine/SigManager.cpp b/bftengine/src/bftengine/SigManager.cpp index 705d55a85c..6e8a84ea2f 100644 --- a/bftengine/src/bftengine/SigManager.cpp +++ b/bftengine/src/bftengine/SigManager.cpp @@ -170,7 +170,9 @@ SigManager::SigManager(PrincipalId myId, } } - clientsPublicKeys_.version = 2; // version `1` suggests RSAVerifier. + /* version `1` suggests RSAVerifier. + * version `2` suggests EdDSAVerifier. */ + clientsPublicKeys_.version = 2; LOG_DEBUG(KEY_EX_LOG, "Map contains " << clientsPublicKeys_.ids_to_keys.size() << " public clients keys"); metrics_component_.Register(); diff --git a/bftengine/src/bftengine/SigManager.hpp b/bftengine/src/bftengine/SigManager.hpp index 095bbce42e..916a8ceda7 100644 --- a/bftengine/src/bftengine/SigManager.hpp +++ b/bftengine/src/bftengine/SigManager.hpp @@ -1,7 +1,7 @@ // Concord // -// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. // // This product is licensed to you under the Apache 2.0 license (the "License"). You may not use this product except in // compliance with the Apache 2.0 License. diff --git a/bftengine/src/bftengine/messages/ViewChangeMsg.cpp b/bftengine/src/bftengine/messages/ViewChangeMsg.cpp index 254a2ec8ff..0261872dcf 100644 --- a/bftengine/src/bftengine/messages/ViewChangeMsg.cpp +++ b/bftengine/src/bftengine/messages/ViewChangeMsg.cpp @@ -162,8 +162,6 @@ void ViewChangeMsg::finalizeMessage() { // | Message Body | // +------------------------------------------+ - // TODO: since there are multiple message with different sequences, make sure that lastStable() - // is the correct value sigManager->sign(lastStable(), body(), bodySize, body() + bodySize); bool b = checkElements((uint16_t)sigSize) && checkComplaints((uint16_t)sigSize); diff --git a/client/reconfiguration/include/client/reconfiguration/poll_based_state_client.hpp b/client/reconfiguration/include/client/reconfiguration/poll_based_state_client.hpp index c19e81ad1b..3f47dd5595 100644 --- a/client/reconfiguration/include/client/reconfiguration/poll_based_state_client.hpp +++ b/client/reconfiguration/include/client/reconfiguration/poll_based_state_client.hpp @@ -24,6 +24,14 @@ namespace concord::client::reconfiguration { +/** + * A polling client which fetches reconfiguration updates from replicas. + * Used by: + * 1. Clients - Clients use this mechanism to get the main key of a replica, which allows + * them to validate tls certificates when replicas replace them. + * 2. Replicas - Replicas in state transfer use this polling client in a best effort + * to execute reconfiguration requests which were missed during ST (e.g wedges, scaling) + */ class PollBasedStateClient : public IStateClient { public: PollBasedStateClient(bft::client::Client* client, diff --git a/tests/apollo/test_skvbc_reconfiguration.py b/tests/apollo/test_skvbc_reconfiguration.py index ba3d889ced..815519d3c3 100644 --- a/tests/apollo/test_skvbc_reconfiguration.py +++ b/tests/apollo/test_skvbc_reconfiguration.py @@ -577,7 +577,7 @@ async def test_tls_exchange_replicas_replicas_with_ror(self, bft_network): for i in range(301): # Produce 301 new blocks await skvbc.send_write_kv_set() # Now, lets switch keys to all replicas - exchanged_replicas = bft_network.all_replicas(without={1,2,3,4,5,6}) + exchanged_replicas = bft_network.all_replicas() await self.run_replica_tls_key_exchange_cycle(bft_network, exchanged_replicas, affected_replicas=[r for r in range(bft_network.config.n + 1)]) bft_network.copy_certs_from_server_to_clients(7) bft_network.restart_clients(False, False) diff --git a/tests/apollo/test_skvbc_state_transfer.py b/tests/apollo/test_skvbc_state_transfer.py index 3110c270b4..5358b1f950 100644 --- a/tests/apollo/test_skvbc_state_transfer.py +++ b/tests/apollo/test_skvbc_state_transfer.py @@ -7,7 +7,7 @@ # # This product may include a number of subcomponents with separate copyright # notices and license terms. Your use of these subcomponents is subject to the -# terms and conditions of the subcomponent's license, as noted in the LICENStE +# terms and conditions of the subcomponent's license, as noted in the LICENSE # file. import os.path import random @@ -287,7 +287,6 @@ async def test_state_transfer_rvt_validity_after_pruning(self, bft_network): # Wait for the RVT root values to be in sync before the pruning await bft_network.wait_for_replicas_rvt_root_values_to_be_in_sync(bft_network.all_replicas()) - return # Get the minimal latest pruneable block among all replicas client = bft_network.random_client() op = operator.Operator(bft_network.config, client, bft_network.builddir) diff --git a/tests/apollo/test_skvbc_view_change.py b/tests/apollo/test_skvbc_view_change.py index 33a2187173..486bd8997e 100644 --- a/tests/apollo/test_skvbc_view_change.py +++ b/tests/apollo/test_skvbc_view_change.py @@ -480,6 +480,8 @@ async def test_multiple_vc_slow_path(self, bft_network, tracker): await bft_network.wait_for_view_with_threshold(view) await skvbc.read_your_writes() + await bft_network.assert_slow_path_prevalent(0, 0, random.choice( + bft_network.all_replicas(without=crashed_replicas))) @with_trio @with_bft_network(start_replica_cmd, diff --git a/tools/KeyfileIOUtils.cpp b/tools/KeyfileIOUtils.cpp index 3492d6acfc..5581277dda 100644 --- a/tools/KeyfileIOUtils.cpp +++ b/tools/KeyfileIOUtils.cpp @@ -135,16 +135,17 @@ Cryptosystem* inputReplicaKeyfileMultisig(const std::string& filename, ReplicaCo if (config.isReadOnly) return nullptr; - // TODO(yf): protect this code with a feature flag - config.thresholdVerificationKeys_.resize(config.numReplicas); - for (auto& [i, hexKey] : config.publicKeysOfReplicas) { - if (i < config.numReplicas) { - config.thresholdVerificationKeys_[i] = hexKey; - LOG_INFO(GL, KVLOG(i, hexKey)); + if (config.singleSignatureScheme) { + config.thresholdVerificationKeys_.resize(config.numReplicas); + for (auto& [i, hexKey] : config.publicKeysOfReplicas) { + if (i < config.numReplicas) { + config.thresholdVerificationKeys_[i] = hexKey; + LOG_INFO(GL, KVLOG(i, hexKey)); + } } - } - config.thresholdPrivateKey_ = config.replicaPrivateKey; + config.thresholdPrivateKey_ = config.replicaPrivateKey; + } return Cryptosystem::fromConfiguration(input, "common", From eabaa78db184a6b989f5f2832b84c1e00b96d82a Mon Sep 17 00:00:00 2001 From: Ynon Flum Date: Sun, 19 Mar 2023 19:50:00 +0200 Subject: [PATCH 13/18] Address additional PR comments --- bftengine/include/bftengine/CryptoManager.hpp | 1 - .../include/bftengine/KeyExchangeManager.hpp | 13 +++- bftengine/src/bftengine/CryptoManager.cpp | 20 ++--- .../src/bftengine/KeyExchangeManager.cpp | 25 ++++-- bftengine/src/bftengine/SigManager.cpp | 77 +++++++++---------- bftengine/src/bftengine/SigManager.hpp | 26 +++---- .../ValidationOnlyIdentityManager.cpp | 2 +- .../tests/SigManager/SigManager_test.cpp | 7 +- kvbc/src/reconfiguration_kvbc_handler.cpp | 3 +- .../threshsign/ThresholdSignaturesTypes.h | 2 +- tests/apollo/util/pyclient/bft_client.py | 1 + 11 files changed, 96 insertions(+), 81 deletions(-) diff --git a/bftengine/include/bftengine/CryptoManager.hpp b/bftengine/include/bftengine/CryptoManager.hpp index e26981d1cc..5fcc52a5c3 100644 --- a/bftengine/include/bftengine/CryptoManager.hpp +++ b/bftengine/include/bftengine/CryptoManager.hpp @@ -134,7 +134,6 @@ class CryptoManager : public IKeyExchanger, public IMultiSigKeyGenerator { void assertMapSizeValid() const; const CheckpointToSystemMap& checkpointToSystem() const; - // chckp -> CryptoSys // TODO: this can be converted to a concurrent queue instead of using a mutex CheckpointToSystemMap cryptoSystems_; // Old cryptosystems can be removed on a checkpoint/cryptosystem creation which might invalidate diff --git a/bftengine/include/bftengine/KeyExchangeManager.hpp b/bftengine/include/bftengine/KeyExchangeManager.hpp index d563f6d1ab..316d59380f 100644 --- a/bftengine/include/bftengine/KeyExchangeManager.hpp +++ b/bftengine/include/bftengine/KeyExchangeManager.hpp @@ -118,6 +118,15 @@ class KeyExchangeManager { // seqnum -> private key std::map keys; + std::optional getGenerationSequenceByPrivateKey(const std::string& hexPrivateKeyToSearch) { + for (auto& [generationSeq, privateKeyHex] : keys) { + if (hexPrivateKeyToSearch == privateKeyHex) { + return generationSeq; + } + } + return std::nullopt; + } + protected: void serializeDataMembers(std::ostream& outStream) const override { serialize(outStream, generated.priv); @@ -143,7 +152,7 @@ class KeyExchangeManager { } // save to secure store void save() { - LOG_INFO(KEY_EX_LOG, "Save keys to " << fs::absolute(secrets_file_)); + LOG_INFO(KEY_EX_LOG, "Persisting keys in file: " << fs::absolute(secrets_file_)); std::stringstream ss; concord::serialize::Serializable::serialize(ss, data_); secretsMgr_->encryptFile(secrets_file_, ss.str()); @@ -207,6 +216,8 @@ class KeyExchangeManager { void waitForLiveQuorum(const ReplicaImp* repImpInstance); void waitForFullCommunication(); void initMetrics(std::shared_ptr a, std::chrono::seconds interval); + // True if the replica's own consensus keys were exchanged + bool exchangedSelfConsensusKeys() const; // deleted KeyExchangeManager(const KeyExchangeManager&) = delete; KeyExchangeManager(const KeyExchangeManager&&) = delete; diff --git a/bftengine/src/bftengine/CryptoManager.cpp b/bftengine/src/bftengine/CryptoManager.cpp index ffe7ff6c75..82ec4b03e1 100644 --- a/bftengine/src/bftengine/CryptoManager.cpp +++ b/bftengine/src/bftengine/CryptoManager.cpp @@ -83,18 +83,18 @@ void CryptoManager::syncPrivateKeyAfterST(const std::string& secretKey, const st void CryptoManager::onPrivateKeyExchange(const std::string& secretKey, const std::string& verificationKey, - const SeqNum& sn) { - LOG_INFO(logger(), "Private key exchange:" << KVLOG(sn, verificationKey)); - auto sys_wrapper = create(sn); + const SeqNum& keyGenerationSn) { + LOG_INFO(logger(), "Private key exchange:" << KVLOG(keyGenerationSn, verificationKey)); + auto sys_wrapper = create(keyGenerationSn); sys_wrapper->cryptosys_->updateKeys(secretKey, verificationKey); sys_wrapper->init(); } void CryptoManager::onPublicKeyExchange(const std::string& verificationKey, const std::uint16_t& signerIndex, - const SeqNum& sn) { - LOG_INFO(logger(), "Public key exchange:" << KVLOG(sn, signerIndex, verificationKey)); - auto sys = create(sn); + const SeqNum& keyGenerationSn) { + LOG_INFO(logger(), "Public key exchange:" << KVLOG(keyGenerationSn, signerIndex, verificationKey)); + auto sys = create(keyGenerationSn); sys->cryptosys_->updateVerificationKey(verificationKey, signerIndex); sys->init(); } @@ -186,13 +186,13 @@ std::shared_ptr CryptoManager::get(const Seq return checkpointToSystem().at(getCheckpointOfCryptosystemForSeq(sn)); } -std::shared_ptr CryptoManager::create(const SeqNum& sn) { +std::shared_ptr CryptoManager::create(const SeqNum& keyGenerationSn) { std::lock_guard guard(mutex_); assertMapSizeValid(); // Cryptosystem for this sn will be activated upon reaching a second checkpoint from now - uint64_t chckp = (sn / checkpointWindowSize) + 2; + CheckpointNum chckp = (keyGenerationSn / checkpointWindowSize) + 2; if (auto it = checkpointToSystem().find(chckp); it != checkpointToSystem().end()) { - LOG_WARN(logger(), "Cryptosystem already exists for checkpoint " << chckp); + LOG_WARN(logger(), "Cryptosystem already exists for checkpoint " << KVLOG(keyGenerationSn, chckp)); return it->second; } @@ -205,7 +205,7 @@ std::shared_ptr CryptoManager::create(const auto ret = insert_result.first->second; LOG_INFO(logger(), "Created new cryptosystem for checkpoint: " << chckp << ", insertion success: " << insert_result.second - << KVLOG(cryptoSystems_.size())); + << KVLOG(cryptoSystems_.size(), keyGenerationSn)); return ret; } diff --git a/bftengine/src/bftengine/KeyExchangeManager.cpp b/bftengine/src/bftengine/KeyExchangeManager.cpp index 3163cccb2d..2045ba50fe 100644 --- a/bftengine/src/bftengine/KeyExchangeManager.cpp +++ b/bftengine/src/bftengine/KeyExchangeManager.cpp @@ -99,7 +99,6 @@ std::string KeyExchangeManager::onKeyExchange(const KeyExchangeMsg& kemsg, const SeqNum& req_sn, const std::string& cid) { const bool isSelfKeyExchange = kemsg.repID == repID_; - bool sentKeyUpdateToExecution = false; SCOPED_MDC_SEQ_NUM(std::to_string(kemsg.generated_sn)); // The key will be used from this sequence forwards // new keys are used two checkpoints after reaching consensus @@ -135,14 +134,14 @@ std::string KeyExchangeManager::onKeyExchange(const KeyExchangeMsg& kemsg, if (ReplicaConfig::instance().getkeyExchangeOnStart() && ReplicaConfig::instance().publishReplicasMasterKeyOnStartup) { sendMainPublicKey(); - sentKeyUpdateToExecution = true; + return "ok"; } } // The key was exchanged successfully, write a block containing the new key to the chain // TODO: need to persist key state and new block atomically // (Cannot be done with current reserved pages implementation) - if (ReplicaConfig::instance().singleSignatureScheme && isSelfKeyExchange && !sentKeyUpdateToExecution) { + if (ReplicaConfig::instance().singleSignatureScheme && isSelfKeyExchange) { sendMainPublicKey(); } return "ok"; @@ -246,7 +245,15 @@ void KeyExchangeManager::exchangeTlsKeys(const SeqNum& bft_sn) { LOG_INFO(KEY_EX_LOG, "Replica communication restarted after tls exchange"); } void KeyExchangeManager::sendMainPublicKey() { - auto [seq, latestPublicKey] = SigManager::instance()->getMyLatestPublicKey(); + constexpr const uint64_t defaultGenerationSeq = 0; + uint64_t generationSeq = defaultGenerationSeq; + auto [latestPrivateKey, latestPublicKey] = SigManager::instance()->getMyLatestKeyPair(); + + if (ReplicaConfig::instance().singleSignatureScheme && exchangedSelfConsensusKeys()) { + generationSeq = + private_keys_.key_data().getGenerationSequenceByPrivateKey(latestPrivateKey).value_or(defaultGenerationSeq); + } + concord::messages::ReconfigurationRequest req; req.sender = repID_; req.command = @@ -254,7 +261,7 @@ void KeyExchangeManager::sendMainPublicKey() { latestPublicKey, "hex", static_cast(SigManager::instance()->getMainKeyAlgorithm()), - static_cast(seq) * checkpointWindowSize}; + generationSeq}; // Mark this request as an internal one std::vector data_vec; concord::messages::serialize(data_vec, req); @@ -265,7 +272,7 @@ void KeyExchangeManager::sendMainPublicKey() { concord::messages::serialize(data_vec, req); std::string cid = "ReplicaMainKeyUpdate_" + std::to_string(repID_); client_->sendRequest(RECONFIG_FLAG, data_vec.size(), reinterpret_cast(data_vec.data()), cid); - LOG_INFO(KEY_EX_LOG, cid + " sent to reconfiguration engine"); + LOG_INFO(KEY_EX_LOG, cid + " sent to reconfiguration engine" << KVLOG(generationSeq, latestPublicKey)); } void KeyExchangeManager::generateConsensusKeyAndSendInternalClientMsg(const SeqNum& sn) { @@ -469,9 +476,13 @@ bool KeyExchangeManager::PrivateKeys::load() { return true; } +bool KeyExchangeManager::exchangedSelfConsensusKeys() const { + return publicKeys_.keyExists(ReplicaConfig::instance().replicaId); +} + bool KeyExchangeManager::isInitialConsensusExchangeComplete() const { uint32_t liveClusterSize = ReplicaConfig::instance().waitForFullCommOnStartup ? clusterSize_ : quorumSize_; - bool exchange_self_keys = publicKeys_.keyExists(ReplicaConfig::instance().replicaId); + bool exchange_self_keys = exchangedSelfConsensusKeys(); LOG_DEBUG(KEY_EX_LOG, KVLOG(ReplicaConfig::instance().waitForFullCommOnStartup, clusterSize_, diff --git a/bftengine/src/bftengine/SigManager.cpp b/bftengine/src/bftengine/SigManager.cpp index 6e8a84ea2f..18bed54fd2 100644 --- a/bftengine/src/bftengine/SigManager.cpp +++ b/bftengine/src/bftengine/SigManager.cpp @@ -212,9 +212,9 @@ uint16_t SigManager::getSigLength(PrincipalId pid) const { } } -bool SigManager::verifyNonReplicaSig( +bool SigManager::verifySigUsingInternalMap( PrincipalId pid, const concord::Byte* data, size_t dataLength, const concord::Byte* sig, uint16_t sigLength) const { - LOG_DEBUG(GL, "Validating NON-replica with: " << KVLOG(myId_, pid, sigLength)); + LOG_DEBUG(GL, "Validating signature using internal map: " << KVLOG(myId_, pid, sigLength)); ConcordAssert(!replicasInfo_.isIdOfReplica(myId_) || !ReplicaConfig::instance().singleSignatureScheme || !replicasInfo_.isIdOfReplica(pid)); bool result = false; @@ -255,18 +255,18 @@ bool SigManager::verifyNonReplicaSig( } } LOG_DEBUG(GL, - "NON-replica validation result: " << KVLOG( + "Internal map validation result: " << KVLOG( result, myId_, pid, sigLength, idOfReadOnlyReplica, idOfExternalClient)); return result; } size_t SigManager::sign(SeqNum seq, const concord::Byte* data, size_t dataLength, concord::Byte* outSig) const { ConcordAssert(replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica()); - concord::crypto::ISigner* rawSigner = nullptr; - std::shared_ptr signer; + const concord::crypto::ISigner* rawSigner = nullptr; + std::shared_ptr signer; if (ReplicaConfig::instance().singleSignatureScheme) { - signer = extractSignerFromMultisig(CryptoManager::instance().getSigner(seq)); - rawSigner = signer.get(); + signer = CryptoManager::instance().getSigner(seq); + rawSigner = &extractSignerFromMultisig(signer); } else { rawSigner = mySigner_.get(); } @@ -279,12 +279,12 @@ size_t SigManager::sign(SeqNum seq, const char* data, size_t dataLength, char* o return sign(seq, reinterpret_cast(data), dataLength, reinterpret_cast(outSig)); } -bool SigManager::verifyReplicaSig(PrincipalId replicaID, - const concord::Byte* data, - size_t dataLength, - const concord::Byte* sig, - uint16_t sigLength) const { - LOG_DEBUG(GL, KVLOG(myId_)); +bool SigManager::verifyReplicaSigUsingMultisigVerifier(PrincipalId replicaID, + const concord::Byte* data, + size_t dataLength, + const concord::Byte* sig, + uint16_t sigLength) const { + ConcordAssert(ReplicaConfig::instance().singleSignatureScheme); ConcordAssert(replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica()); ConcordAssert(replicasInfo_.isIdOfReplica(replicaID)); int lastVerifierIndex = 0; @@ -295,32 +295,32 @@ bool SigManager::verifyReplicaSig(PrincipalId replicaID, } auto& verifier = extractVerifierFromMultisig(multisigVerifier, replicaID); LOG_DEBUG(GL, - "Validating as replica with: " << KVLOG( + "Validating replica signature with: " << KVLOG( myId_, replicaID, lastVerifierIndex, verifier.getPubKey(), sigLength, verifier.signatureLength())); if (verifier.verifyBuffer(data, dataLength, sig, sigLength)) { LOG_DEBUG(GL, - "Validation Successful " << KVLOG( - myId_, replicaID, lastVerifierIndex, sigLength, verifier.signatureLength())); + "Replica signature validation Successful " + << KVLOG(myId_, replicaID, lastVerifierIndex, sigLength, verifier.signatureLength())); return true; } else { LOG_DEBUG(GL, - "Validation failed as replica with: " << KVLOG( + "Replica signature validation failed: " << KVLOG( myId_, replicaID, lastVerifierIndex, sigLength, verifier.signatureLength())); } lastVerifierIndex++; } - LOG_WARN(GL, "Validation failed with all cryptosystems" << KVLOG(myId_, replicaID, lastVerifierIndex)); + LOG_WARN(GL, "Validation failed using all cryptosystems" << KVLOG(myId_, replicaID, lastVerifierIndex)); return false; } // verify using the two last keys, once the last key's checkpoint is reached, the previous key is removed bool SigManager::verifySig( PrincipalId pid, const concord::Byte* data, size_t dataLength, const concord::Byte* sig, uint16_t sigLength) const { - if (replicasInfo_.isIdOfReplica(pid) && ReplicaConfig::instance().singleSignatureScheme) { - return verifyReplicaSig(pid, data, dataLength, sig, sigLength); + if (ReplicaConfig::instance().singleSignatureScheme && replicasInfo_.isIdOfReplica(pid)) { + return verifyReplicaSigUsingMultisigVerifier(pid, data, dataLength, sig, sigLength); } - return verifyNonReplicaSig(pid, data, dataLength, sig, sigLength); + return verifySigUsingInternalMap(pid, data, dataLength, sig, sigLength); } bool SigManager::verifyOwnSignature(const concord::Byte* data, @@ -330,7 +330,7 @@ bool SigManager::verifyOwnSignature(const concord::Byte* data, if (ReplicaConfig::instance().singleSignatureScheme) { auto signers = CryptoManager::instance().getLatestSigners(); for (auto& signer : signers) { - extractSignerFromMultisig(signer)->signBuffer(data, dataLength, sig.data()); + extractSignerFromMultisig(signer).signBuffer(data, dataLength, sig.data()); if (std::memcmp(sig.data(), expectedSignature, getMySigLength()) == 0) { LOG_DEBUG(GL, "Self-sig validation succeeded"); @@ -346,32 +346,25 @@ bool SigManager::verifyOwnSignature(const concord::Byte* data, return std::memcmp(sig.data(), expectedSignature, getMySigLength()) == 0; } -std::shared_ptr SigManager::extractSignerFromMultisig( +const concord::crypto::ISigner& SigManager::extractSignerFromMultisig( std::shared_ptr thresholdSigner) const { ConcordAssert(ReplicaConfig::instance().singleSignatureScheme); ConcordAssertNE(thresholdSigner.get(), nullptr); - std::shared_ptr eddsaSigner = std::dynamic_pointer_cast(thresholdSigner); - ConcordAssertNE(eddsaSigner.get(), nullptr); - std::shared_ptr baseSigner = - std::dynamic_pointer_cast(eddsaSigner); - ConcordAssertNE(baseSigner.get(), nullptr); - return baseSigner; + return reinterpret_cast(*thresholdSigner.get()); } const concord::crypto::IVerifier& SigManager::extractVerifierFromMultisig( std::shared_ptr thresholdVerifier, PrincipalId id) const { ConcordAssert(ReplicaConfig::instance().singleSignatureScheme); ConcordAssertNE(thresholdVerifier.get(), nullptr); - std::shared_ptr eddsaVerifier = - std::dynamic_pointer_cast(thresholdVerifier); - ConcordAssertNE(eddsaVerifier.get(), nullptr); - return eddsaVerifier->getVerifier(id); + return reinterpret_cast(*thresholdVerifier.get()).getVerifier(id); } uint16_t SigManager::getMySigLength() const { if (ReplicaConfig::instance().singleSignatureScheme && (replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica())) { - return extractSignerFromMultisig(getCurrentReplicaSigner())->signatureLength(); + auto currentSigner = getCurrentReplicaSigner(); + return extractSignerFromMultisig(currentSigner).signatureLength(); } return (uint16_t)mySigner_->signatureLength(); } @@ -404,22 +397,24 @@ std::shared_ptr SigManager::getCurrentReplicaSigner() const { return signer; } -std::pair SigManager::getMyLatestPublicKey() const { +std::pair SigManager::getMyLatestKeyPair() const { ConcordAssert(replicasInfo_.isIdOfReplica(myId_)); if (!ReplicaConfig::instance().singleSignatureScheme) { - return {0, getVerifier(myId_).getPubKey()}; + return {mySigner_->getPrivKey(), getVerifier(myId_).getPubKey()}; } - auto [seq, latestVerifier] = CryptoManager::instance().getLatestVerifiers()[0]; - auto pubKey = extractVerifierFromMultisig(latestVerifier, myId_).getPubKey(); - LOG_DEBUG(GL, KVLOG(seq, pubKey)); - return {seq, pubKey}; + auto& system = CryptoManager::instance().getLatestCryptoSystem(); + std::pair ret = {system->getPrivateKey(replicasInfo_.myId()), + system->getMyVerificationKey()}; + LOG_INFO(GL, KVLOG(ret.first, ret.second)); + return ret; } std::string SigManager::getSelfPrivKey() const { if (ReplicaConfig::instance().singleSignatureScheme) { - return extractSignerFromMultisig(getCurrentReplicaSigner())->getPrivKey(); + auto currentSigner = getCurrentReplicaSigner(); + return extractSignerFromMultisig(currentSigner).getPrivKey(); } return mySigner_->getPrivKey(); } diff --git a/bftengine/src/bftengine/SigManager.hpp b/bftengine/src/bftengine/SigManager.hpp index 916a8ceda7..63bb5f307e 100644 --- a/bftengine/src/bftengine/SigManager.hpp +++ b/bftengine/src/bftengine/SigManager.hpp @@ -86,11 +86,11 @@ class SigManager { size_t sign(SeqNum seq, const concord::Byte* data, size_t dataLength, concord::Byte* outSig) const; size_t sign(SeqNum seq, const char* data, size_t dataLength, char* outSig) const; - bool verifyReplicaSig(PrincipalId replicaID, - const concord::Byte* data, - size_t dataLength, - const concord::Byte* sig, - uint16_t sigLength) const; + bool verifyReplicaSigUsingMultisigVerifier(PrincipalId replicaID, + const concord::Byte* data, + size_t dataLength, + const concord::Byte* sig, + uint16_t sigLength) const; uint16_t getMySigLength() const; bool isClientTransactionSigningEnabled() { return clientTransactionSigningEnabled_; } @@ -110,13 +110,14 @@ class SigManager { std::string getClientsPublicKeys(); // Hex format - std::pair getMyLatestPublicKey() const; + std::pair getMyLatestKeyPair() const; // Used by AsyncTLSConnection to verify tls certificates which replicas sign using their main key // Up to replicaIdentityHistoryCount keys are returned by replicas std::array getPublicKeyOfVerifier(uint32_t id) const; // Used only by replicas + // Returns the private key which corresponds to the last executed sequence std::string getSelfPrivKey() const; void setReplicaLastExecutedSeq(SeqNum seq); @@ -124,8 +125,7 @@ class SigManager { bool verifyOwnSignature(const concord::Byte* data, size_t dataLength, const concord::Byte* expectedSignature) const; - std::shared_ptr extractSignerFromMultisig( - std::shared_ptr thresholdSigner) const; + const concord::crypto::ISigner& extractSignerFromMultisig(std::shared_ptr thresholdSigner) const; const concord::crypto::IVerifier& extractVerifierFromMultisig(std::shared_ptr thresholdVerifier, PrincipalId id) const; @@ -140,11 +140,11 @@ class SigManager { const std::optional>& operatorKey, const ReplicasInfo& replicasInfo); - bool verifyNonReplicaSig(PrincipalId pid, - const concord::Byte* data, - size_t dataLength, - const concord::Byte* sig, - uint16_t sigLength) const; + bool verifySigUsingInternalMap(PrincipalId pid, + const concord::Byte* data, + size_t dataLength, + const concord::Byte* sig, + uint16_t sigLength) const; std::shared_ptr getCurrentReplicaSigner() const; diff --git a/bftengine/src/bftengine/ValidationOnlyIdentityManager.cpp b/bftengine/src/bftengine/ValidationOnlyIdentityManager.cpp index cbec778b5e..49c71a9a26 100644 --- a/bftengine/src/bftengine/ValidationOnlyIdentityManager.cpp +++ b/bftengine/src/bftengine/ValidationOnlyIdentityManager.cpp @@ -29,7 +29,7 @@ ValidationOnlyIdentityManager::ValidationOnlyIdentityManager( bool ValidationOnlyIdentityManager::verifySig( PrincipalId pid, const concord::Byte* data, size_t dataLength, const concord::Byte* sig, uint16_t sigLength) const { - return verifyNonReplicaSig(pid, data, dataLength, sig, sigLength); + return verifySigUsingInternalMap(pid, data, dataLength, sig, sigLength); } // This method is assumed to be called by a single thread diff --git a/bftengine/tests/SigManager/SigManager_test.cpp b/bftengine/tests/SigManager/SigManager_test.cpp index 40f6cbe6fd..274c1324bc 100644 --- a/bftengine/tests/SigManager/SigManager_test.cpp +++ b/bftengine/tests/SigManager/SigManager_test.cpp @@ -27,11 +27,9 @@ constexpr size_t RANDOM_DATA_SIZE = 1000U; std::default_random_engine generator; using concord::crypto::ISigner; -using concord::crypto::IVerifier; using concord::crypto::Factory; using bftEngine::ReplicaConfig; using bftEngine::CryptoManager; -using concord::crypto::SignatureAlgorithm; using concord::crypto::generateEdDSAKeyPair; std::vector> generateKeyPairs(size_t count) { @@ -160,7 +158,8 @@ class SigManagerTest : public ::testing::Test { consensusHexReplicaPublicKeys = hexReplicaPublicKeys; } else { consensusHexKeyPairs = generateKeyPairs(config.numReplicas); - for (auto& [privateKey, publicKey] : consensusHexKeyPairs) { + for (auto& pair : consensusHexKeyPairs) { + auto& publicKey = pair.second; consensusHexReplicaPublicKeys.push_back(publicKey); } } @@ -190,7 +189,7 @@ class MultiSchemeSigManagerTest : public SigManagerTest { }; TEST_F(SingleSchemeSigManagerTest, TestSignerExtraction) { - ASSERT_EQ(sigManager->extractSignerFromMultisig(cryptoManager->getSigner(0))->getPrivKey(), hexKeyPairs[0].first); + ASSERT_EQ(sigManager->extractSignerFromMultisig(cryptoManager->getSigner(0)).getPrivKey(), hexKeyPairs[0].first); } TEST_F(SingleSchemeSigManagerTest, TestVerifierExtraction) { diff --git a/kvbc/src/reconfiguration_kvbc_handler.cpp b/kvbc/src/reconfiguration_kvbc_handler.cpp index 71c688c8af..7750fc392f 100644 --- a/kvbc/src/reconfiguration_kvbc_handler.cpp +++ b/kvbc/src/reconfiguration_kvbc_handler.cpp @@ -102,7 +102,7 @@ kvbc::BlockId ReconfigurationBlockTools::persistReconfigurationBlock( try { auto ret = blocks_adder_.add(std::move(updates)); - LOG_INFO(GL, "Persist result: " << KVLOG(ret)); + LOG_DEBUG(GL, "Persist result: " << KVLOG(ret)); return ret; } catch (const std::exception& e) { LOG_ERROR(GL, "failed to persist the reconfiguration block: " << e.what()); @@ -443,7 +443,6 @@ concord::messages::ClientStateReply KvbcClientReconfigurationHandler::buildRepli } else if (command_type == std::string{kvbc::keyTypes::reconfiguration_rep_main_key}) { concord::messages::ReplicaMainKeyUpdate cmd; concord::messages::deserialize(data_buf, cmd); - cmd.seq_num -= (cmd.seq_num < 2 * checkpointWindowSize ? 0 : 2 * checkpointWindowSize); creply.response = cmd; } auto epoch_data = diff --git a/libs/crypto/threshsign/ThresholdSignaturesTypes.h b/libs/crypto/threshsign/ThresholdSignaturesTypes.h index 3b3c611d1a..a6b79959ee 100644 --- a/libs/crypto/threshsign/ThresholdSignaturesTypes.h +++ b/libs/crypto/threshsign/ThresholdSignaturesTypes.h @@ -320,7 +320,7 @@ class Cryptosystem { std::string& thrPublicKey, std::vector& thrVerificationKeys); /** - * Update a key pair + * Update own key pair */ void updateKeys(const std::string& shareSecretKey, const std::string& shareVerificationKey); diff --git a/tests/apollo/util/pyclient/bft_client.py b/tests/apollo/util/pyclient/bft_client.py index 514efc1567..31ccb2fc1b 100644 --- a/tests/apollo/util/pyclient/bft_client.py +++ b/tests/apollo/util/pyclient/bft_client.py @@ -123,6 +123,7 @@ def All(cls, config, replicas): def __str__(self): return f"n: {self.replicas}, m: {self.required}" + class BftClient(ABC): COUNTER = WriteCounter(val=int(1e6)) From 6473a9c87c031333d095e3fe4aa02026db55f688 Mon Sep 17 00:00:00 2001 From: Ynon Flum Date: Tue, 21 Mar 2023 18:02:03 +0200 Subject: [PATCH 14/18] Persist recovered private keys on ST, address additional PR comments --- bftengine/include/bftengine/CryptoManager.hpp | 15 ++-- .../include/bftengine/KeyExchangeManager.hpp | 18 +++-- .../bcstatetransfer/AsyncStateTransferCRE.hpp | 8 --- bftengine/src/bftengine/CryptoManager.cpp | 32 ++++++--- .../src/bftengine/KeyExchangeManager.cpp | 69 ++++++++++++------- .../src/bftengine/ReplicaForStateTransfer.cpp | 42 ++++++----- bftengine/src/bftengine/SigManager.cpp | 10 ++- libs/util/assertUtils.hpp | 2 +- libs/util/containers.hpp | 12 ++++ .../TesterReplica/internalCommandsHandler.cpp | 12 ++-- 10 files changed, 136 insertions(+), 84 deletions(-) create mode 100644 libs/util/containers.hpp diff --git a/bftengine/include/bftengine/CryptoManager.hpp b/bftengine/include/bftengine/CryptoManager.hpp index 5fcc52a5c3..65f7b69a40 100644 --- a/bftengine/include/bftengine/CryptoManager.hpp +++ b/bftengine/include/bftengine/CryptoManager.hpp @@ -13,6 +13,7 @@ #include #include +#include #include "log/logger.hpp" #include "crypto/threshsign/ThresholdSignaturesTypes.h" @@ -69,16 +70,16 @@ class CryptoManager : public IKeyExchanger, public IMultiSigKeyGenerator { const SeqNum& sn) override; /** - * Iterates saved cryptosystems and updates their private key if their public key matches verificationKey - * After ST, if a replica has not executed its key exchange but the network did, the internal private key state needs - * to be synchronized. - * @param secretKey - * @param verificationKey + * Synchronizes the private keys of existing cryptosystems with the candidate state from KeyExchangeManager + * After ST. If a replica has not executed a key exchange it had previously initiated, + * but the network did execute it, the internal private key state needs to be synchronized. + * @param candidateKeys - The keys for which a cryptosystem has yet to be created + * @return A map containing candidates to persist * @note: Assumes all keys are formatted as hex strings * @note: TODO: Current implementation is not crash consistent, a ST which was completed after the termination - * of the replica process will lose the new private key + * of the replica process will result in the loss of a new private key */ - void syncPrivateKeyAfterST(const std::string& secretKey, const std::string& verificationKey); + std::set syncPrivateKeysAfterST(const std::map>& candidateKeys); void onCheckpoint(uint64_t newCheckpoint); diff --git a/bftengine/include/bftengine/KeyExchangeManager.hpp b/bftengine/include/bftengine/KeyExchangeManager.hpp index 316d59380f..bd68a81044 100644 --- a/bftengine/include/bftengine/KeyExchangeManager.hpp +++ b/bftengine/include/bftengine/KeyExchangeManager.hpp @@ -45,13 +45,13 @@ class KeyExchangeManager { // The new key pair will be used from two checkpoints after kemsg.generated_sn std::string onKeyExchange(const KeyExchangeMsg& kemsg, const SeqNum& req_sn, const std::string& cid); /** - * Updates new key pair in both volatile (CryptoManager) and persistent (Reserved pages) memory + * Updates new key pair in both volatile (CryptoManager) and persistent (reserved pages) memory * @param repID - The id of the replica which uses pubkey - * @param effective_key_sn - The sequence number from which the new key is to be used + * @param keyGenerationSn - The sequence number in which the key was generated * @param pubkey - The public key of replica repID * @param cid */ - void registerNewKeyPair(uint16_t repID, SeqNum effective_key_sn, const std::string& pubkey, const std::string& cid); + void registerNewKeyPair(uint16_t repID, SeqNum keyGenerationSn, const std::string& pubkey, const std::string& cid); // Register a IKeyExchanger to notification when keys are rotated. void registerForNotification(IKeyExchanger* ke) { registryToExchange_.push_back(ke); } // Called at the end of state transfer @@ -81,7 +81,8 @@ class KeyExchangeManager { NodeIdType clientId, bool saveToReservedPages); - std::pair getCandidateKeyPair() const; + void persistCandidates(const std::set& candidatesToPersist); + ///////// end - Clients public keys interface/////////////// std::string getStatus() const; @@ -94,7 +95,7 @@ class KeyExchangeManager { public: // internal persistent private keys impl struct KeyData : public concord::serialize::SerializableFactory { - struct { + struct GeneratedKeyPairInfo { // generated private key std::string priv; // generated public key @@ -115,6 +116,8 @@ class KeyExchangeManager { algorithm = concord::crypto::SignatureAlgorithm::Uninitialized; } } generated; + // TODO: A map containing all generated candidates should replace this single candidate. + // seqnum -> private key std::map keys; @@ -203,6 +206,8 @@ class KeyExchangeManager { return km; } + std::map> getCandidates() const; + private: // methods KeyExchangeManager(InitData* id); std::string generateCid(std::string); @@ -230,7 +235,6 @@ class KeyExchangeManager { uint32_t quorumSize_{}; ClusterKeyStore publicKeys_; PrivateKeys private_keys_; - PrivateKeys::KeyData candidate_private_keys_; ClientKeyStore clientsPublicKeys_; // A flag to prevent race on the replica's internal client. std::atomic_bool initial_exchange_; @@ -242,7 +246,7 @@ class KeyExchangeManager { bool publishedMasterKey = false; std::mutex startup_mutex_; // map to store seqNum and its candidate key - std::map seq_candidate_map_; + std::map seq_candidate_map_; struct Metrics { std::chrono::seconds lastMetricsDumpTime; diff --git a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.hpp b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.hpp index 78bedac00d..cce75f1169 100644 --- a/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.hpp +++ b/bftengine/src/bcstatetransfer/AsyncStateTransferCRE.hpp @@ -23,14 +23,6 @@ namespace bftEngine::bcst::asyncCRE { class CreFactory { public: - /* - * TODO: Transaction signing needs to be disabled for replicas, as state transfer may change - * a replica's consensus (and therefore main) key. Apollo tests currently run with transaction signing enabled, - * we therefore use the latest key published by this replica. - * If a consensus was reached over a key exchange which was initiated by this replica - * prior to issuing CRE client requests, the other replicas are guaranteed to be able to use - * this replica's latest main key assuming an honest execution of the exchange. - */ static std::shared_ptr create( std::shared_ptr msgsCommunicator, std::shared_ptr msgHandlers, diff --git a/bftengine/src/bftengine/CryptoManager.cpp b/bftengine/src/bftengine/CryptoManager.cpp index 82ec4b03e1..e80aaace96 100644 --- a/bftengine/src/bftengine/CryptoManager.cpp +++ b/bftengine/src/bftengine/CryptoManager.cpp @@ -11,7 +11,7 @@ #include "ReplicaConfig.hpp" #include "CryptoManager.hpp" -#include +#include "util/containers.hpp" namespace bftEngine { @@ -69,16 +69,27 @@ std::tuple Crypto return {priv, pub, getLatestSignatureAlgorithm()}; } -void CryptoManager::syncPrivateKeyAfterST(const std::string& secretKey, const std::string& verificationKey) { +std::set CryptoManager::syncPrivateKeysAfterST( + const std::map>& candidateKeys) { + ConcordAssert(ReplicaConfig::instance().singleSignatureScheme); std::lock_guard guard(mutex_); - for (auto& [checkpoint, cryptoSystem] : checkpointToSystem()) { - auto currentKey = cryptoSystem->cryptosys_->getMyVerificationKey(); - LOG_INFO(logger(), "checking keys after ST" << KVLOG(checkpoint, currentKey, secretKey, verificationKey)); - if (currentKey == verificationKey) { - cryptoSystem->cryptosys_->updateKeys(secretKey, verificationKey); - LOG_INFO(logger(), "Updated private key for cryptosystem with checkpoint:" << checkpoint); + std::set candidatesToPersist; + for (auto& [keyGenerationSn, keyPair] : candidateKeys) { + auto effeciveCheckpoint = (keyGenerationSn / checkpointWindowSize) + 2; + if (checkpointToSystem().find(effeciveCheckpoint) != checkpointToSystem().end()) { + auto& cryptoSystem = checkpointToSystem().at(effeciveCheckpoint)->cryptosys_; + auto& [privateKey, publicKey] = keyPair; + LOG_INFO(logger(), + "Found candidate matching the same checkpoint as an existing cryptoSystem" + << KVLOG(keyGenerationSn, effeciveCheckpoint, publicKey, cryptoSystem->getMyVerificationKey())); + if (cryptoSystem->getMyVerificationKey() == publicKey) { + cryptoSystem->updateKeys(privateKey, publicKey); + candidatesToPersist.emplace(keyGenerationSn); + LOG_INFO(logger(), "Updated private key for cryptosystem with checkpoint:" << effeciveCheckpoint); + } } } + return candidatesToPersist; } void CryptoManager::onPrivateKeyExchange(const std::string& secretKey, @@ -104,7 +115,7 @@ void CryptoManager::onCheckpoint(uint64_t newCheckpoint) { std::vector checkpointsToRemove; auto checkpointOfCurrentCryptoSystem = getCheckpointOfCryptosystemForSeq(newCheckpoint * checkpointWindowSize); - std::experimental::erase_if( + concord::util::containers::erase_if( cryptoSystems_, [this, checkpointOfCurrentCryptoSystem, newCheckpoint](const auto& checkpointToSystem) { auto checkpoint = checkpointToSystem.first; if (checkpoint < checkpointOfCurrentCryptoSystem) { @@ -197,6 +208,9 @@ std::shared_ptr CryptoManager::create(const } // copy construct new Cryptosystem from a last one as we want it to include all the existing keys + // Cryptosystems are created either by key exchanges or ST, in ST they are rebuilt in ascending sequence order + // If a key exchange execution results in copying the wrong cryptosystem, the key state will be rebuilt + // in the next ST cycle std::unique_ptr cs = std::make_unique(*checkpointToSystem().rbegin()->second->cryptosys_.get()); diff --git a/bftengine/src/bftengine/KeyExchangeManager.cpp b/bftengine/src/bftengine/KeyExchangeManager.cpp index 2045ba50fe..5dd6768dde 100644 --- a/bftengine/src/bftengine/KeyExchangeManager.cpp +++ b/bftengine/src/bftengine/KeyExchangeManager.cpp @@ -73,25 +73,25 @@ std::string KeyExchangeManager::generateCid(std::string cid) { } void KeyExchangeManager::registerNewKeyPair(uint16_t repID, - SeqNum sn, + SeqNum keyGenerationSn, const std::string& pubkey, const std::string& cid) { const bool isSelfKeyExchange = repID == repID_; - publicKeys_.push(repID, pubkey, sn); + publicKeys_.push(repID, pubkey, keyGenerationSn); if (!isSelfKeyExchange) { - for (auto* e : registryToExchange_) e->onPublicKeyExchange(pubkey, repID, sn); + for (auto* e : registryToExchange_) e->onPublicKeyExchange(pubkey, repID, keyGenerationSn); metrics_->public_key_exchange_for_peer_counter++; return; } - private_keys_.key_data().generated = seq_candidate_map_[sn].generated; - seq_candidate_map_[sn].generated.clear(); - candidate_private_keys_.generated.clear(); - // erasing seqnum from the map - seq_candidate_map_.erase(sn); + ConcordAssertNE(seq_candidate_map_.find(keyGenerationSn), seq_candidate_map_.end()); + private_keys_.key_data().generated = seq_candidate_map_[keyGenerationSn]; + seq_candidate_map_.erase(keyGenerationSn); + // TODO: Figure out when old candidates can be removed ConcordAssert(private_keys_.key_data().generated.pub == pubkey); - private_keys_.onKeyExchange(cid, sn); - for (auto e : registryToExchange_) e->onPrivateKeyExchange(private_keys_.key_data().keys[sn], pubkey, sn); + private_keys_.onKeyExchange(cid, keyGenerationSn); + for (auto e : registryToExchange_) + e->onPrivateKeyExchange(private_keys_.key_data().keys[keyGenerationSn], pubkey, keyGenerationSn); metrics_->self_key_exchange_counter++; } @@ -122,6 +122,10 @@ std::string KeyExchangeManager::onKeyExchange(const KeyExchangeMsg& kemsg, } if (publicKeys_.keyExists(kemsg.repID, sn)) return "ok"; + if (isSelfKeyExchange && seq_candidate_map_.find(sn) == seq_candidate_map_.end()) { + return "missing_candidate"; + } + registerNewKeyPair(kemsg.repID, sn, kemsg.pubkey, cid); auto liveQuorumSize = ReplicaConfig::instance().waitForFullCommOnStartup ? clusterSize_ : quorumSize_; @@ -283,14 +287,11 @@ void KeyExchangeManager::generateConsensusKeyAndSendInternalClientMsg(const SeqN return; } - auto& candidate = candidate_private_keys_.generated; - bool generateNewPair = candidate.cid.empty() && (seq_candidate_map_.count(sn) == 0); - KeyExchangeMsg msg; - msg.repID = repID_; - msg.generated_sn = sn; - msg.epoch = EpochManager::instance().getSelfEpochNumber(); + bool generateNewPair = seq_candidate_map_.find(sn) == seq_candidate_map_.end(); if (generateNewPair) { + seq_candidate_map_[sn] = {}; + auto& candidate = seq_candidate_map_[sn]; auto cid = generateCid(kInitialKeyExchangeCid); auto [prv, pub, algorithm] = multiSigKeyHdlr_->generateMultisigKeyPair(); candidate.algorithm = algorithm; @@ -298,19 +299,23 @@ void KeyExchangeManager::generateConsensusKeyAndSendInternalClientMsg(const SeqN candidate.pub = pub; candidate.cid = cid; candidate.sn = sn; - seq_candidate_map_[sn] = candidate_private_keys_; LOG_INFO(KEY_EX_LOG, "Added new candidate" << KVLOG(candidate.cid, sn)); } else { - LOG_INFO( - KEY_EX_LOG, - "we already have a candidate for this sequence number, trying to send it again" << KVLOG(candidate.cid, sn)); - candidate_private_keys_.generated.sn = sn; - seq_candidate_map_[sn] = candidate_private_keys_; + LOG_INFO(KEY_EX_LOG, + "we already have a candidate for this sequence number, trying to send it again" + << KVLOG(seq_candidate_map_.at(sn).cid, sn)); } + auto& candidate = seq_candidate_map_.at(sn); + KeyExchangeMsg msg; + msg.repID = repID_; + msg.generated_sn = candidate.sn; + msg.epoch = EpochManager::instance().getSelfEpochNumber(); msg.pubkey = candidate.pub; msg.algorithm = candidate.algorithm; - LOG_INFO(KEY_EX_LOG, "Sending consensus key exchange :" << KVLOG(sn, candidate.cid, msg.pubkey, msg.algorithm)); + LOG_INFO(KEY_EX_LOG, + "Sending consensus key exchange message:" << KVLOG( + candidate.sn, candidate.cid, msg.pubkey, msg.algorithm, generateNewPair)); client_->sendRequest(bftEngine::KEY_EXCHANGE_FLAG, msg, candidate.cid); metrics_->sent_key_exchange_counter++; } @@ -494,9 +499,21 @@ bool KeyExchangeManager::isInitialConsensusExchangeComplete() const { ? (publicKeys_.numOfExchangedReplicas() + 1 >= liveClusterSize) && exchange_self_keys : true; } -std::pair KeyExchangeManager::getCandidateKeyPair() const { - auto& candidate = candidate_private_keys_.generated; - return {candidate.priv, candidate.pub}; + +std::map> KeyExchangeManager::getCandidates() const { + std::map> result; + for (auto& [seq, generatedPairInfo] : seq_candidate_map_) { + result[seq] = {generatedPairInfo.priv, generatedPairInfo.pub}; + } + return result; +} + +void KeyExchangeManager::persistCandidates(const std::set& candidatesToPersist) { + for (auto keyGenerationSn : candidatesToPersist) { + auto& keyPairInfo = seq_candidate_map_[keyGenerationSn]; + LOG_INFO(KEY_EX_LOG, "Persisting candidate's private key" << KVLOG(keyGenerationSn, keyPairInfo.pub)); + registerNewKeyPair(repID_, keyPairInfo.sn, keyPairInfo.pub, keyPairInfo.cid); + } } } // namespace bftEngine::impl diff --git a/bftengine/src/bftengine/ReplicaForStateTransfer.cpp b/bftengine/src/bftengine/ReplicaForStateTransfer.cpp index 29a44f71c1..dbf2518b8e 100644 --- a/bftengine/src/bftengine/ReplicaForStateTransfer.cpp +++ b/bftengine/src/bftengine/ReplicaForStateTransfer.cpp @@ -79,22 +79,29 @@ void ReplicaForStateTransfer::start() { stateTransfer->addOnTransferringCompleteCallback( [this](std::uint64_t checkpoint) { // TODO - The next lines up to comment 'YYY' do not belong here (CRE) - consider refactor or move outside + // RO replica relies on async CRE to receive key state updates if (!config_.isReadOnly) { - // Load the public keys of the other replicas from reserved pages - // so that their responses can be validated - KeyExchangeManager::instance().loadPublicKeys(); - - // Make sure to sign the reconfiguration client messages using the key - // other replicas expect - SigManager::instance()->setReplicaLastExecutedSeq(checkpoint * checkpointWindowSize); - - // Need to update private key to match the loaded public key in case they differ (key exchange was executed - // on other replicas but not on this one, finishing ST does not mean that missed key exchanges are executed) - // This can be done by iterating the saved cryptosystems and updating their private key if their - // public key matches the candidate saved in KeyExchangeManager - CryptoManager::instance().onCheckpoint(checkpoint); - auto [priv, pub] = KeyExchangeManager::instance().getCandidateKeyPair(); - CryptoManager::instance().syncPrivateKeyAfterST(priv, pub); + if (ReplicaConfig::instance().singleSignatureScheme) { + auto &keyExchangeManager = KeyExchangeManager::instance(); + auto &cryptoManager = CryptoManager::instance(); + auto &sigManager = *SigManager::instance(); + // Load the public keys of the other replicas from reserved pages + // so that their responses can be validated + keyExchangeManager.loadPublicKeys(); + + // Make sure to sign the reconfiguration client messages using the key + // other replicas expect + sigManager.setReplicaLastExecutedSeq(checkpoint * checkpointWindowSize); + + // Need to update private key to match the loaded public key in case they differ (key exchange was executed + // on other replicas but not on this one, finishing ST does not mean that missed key exchanges are executed) + // This can be done by iterating the saved cryptosystems and updating their private key if their + // public key matches the candidate saved in KeyExchangeManager + cryptoManager.onCheckpoint(checkpoint); + + auto candidatesToPersist = cryptoManager.syncPrivateKeysAfterST(keyExchangeManager.getCandidates()); + keyExchangeManager.persistCandidates(candidatesToPersist); + } // At this point, we, if are not going to have another blocks in state transfer. So, we can safely stop CRE. // if there is a reconfiguration state change that prevents us from starting another state transfer (i.e. @@ -103,8 +110,9 @@ void ReplicaForStateTransfer::start() { auto *pbc = reinterpret_cast(cre_->getStateClient()); - // TODO: remove loop so that state transfer doesn't hang if it cannot complete reconfiguration requests - // The current implementation expects f + 1 identical responses + /* TODO: remove loop so that state transfer doesn't hang if it cannot complete reconfiguration requests, + as it is assumed to require only a valid communication channel + The current implementation expects f + 1 identical responses */ bool succ = false; while (!succ) { auto latestHandledUpdate = cre_->getLatestKnownUpdateBlock(); diff --git a/bftengine/src/bftengine/SigManager.cpp b/bftengine/src/bftengine/SigManager.cpp index 18bed54fd2..14ed5df1e8 100644 --- a/bftengine/src/bftengine/SigManager.cpp +++ b/bftengine/src/bftengine/SigManager.cpp @@ -386,7 +386,12 @@ void SigManager::setClientPublicKey(const std::string& key, PrincipalId id, KeyF LOG_WARN(KEY_EX_LOG, "Illegal id for client " << id); } } -bool SigManager::hasVerifier(PrincipalId pid) { return verifiers_.find(pid) != verifiers_.end(); } +bool SigManager::hasVerifier(PrincipalId pid) { + if (ReplicaConfig::instance().singleSignatureScheme && replicasInfo_.isIdOfReplica(pid)) { + return true; + } + return verifiers_.find(pid) != verifiers_.end(); +} concord::crypto::SignatureAlgorithm SigManager::getMainKeyAlgorithm() const { return concord::crypto::EdDSA; } @@ -407,7 +412,6 @@ std::pair SigManager::getMyLatestKeyPair() const { auto& system = CryptoManager::instance().getLatestCryptoSystem(); std::pair ret = {system->getPrivateKey(replicasInfo_.myId()), system->getMyVerificationKey()}; - LOG_INFO(GL, KVLOG(ret.first, ret.second)); return ret; } @@ -445,7 +449,7 @@ const concord::crypto::IVerifier& SigManager::getVerifier(PrincipalId otherPrinc } void SigManager::setReplicaLastExecutedSeq(SeqNum seq) { - ConcordAssert(replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica()); + ConcordAssert(replicasInfo_.isIdOfReplica(myId_)); replicaLastExecutedSeq_ = seq; } diff --git a/libs/util/assertUtils.hpp b/libs/util/assertUtils.hpp index b42ba0cfa8..2efcc1a3cb 100644 --- a/libs/util/assertUtils.hpp +++ b/libs/util/assertUtils.hpp @@ -60,7 +60,7 @@ inline void printCallStack() { free(ret); } } - LOG_INFO(GL, "\n" << os.str()); + LOG_FATAL(GL, "\n" << os.str()); std::free(symbolsList); } } diff --git a/libs/util/containers.hpp b/libs/util/containers.hpp new file mode 100644 index 0000000000..aecb535edd --- /dev/null +++ b/libs/util/containers.hpp @@ -0,0 +1,12 @@ +namespace concord::util::containers { +template +void erase_if(Container& items, const Predicate& predicate) { + for (auto iter = items.begin(); iter != items.end();) { + if (predicate(*iter)) { + iter = items.erase(iter); + } else { + ++iter; + } + } +} +} // namespace concord::util::containers \ No newline at end of file diff --git a/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp b/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp index 7c68e2094c..18dee2d796 100644 --- a/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp +++ b/tests/simpleKVBC/TesterReplica/internalCommandsHandler.cpp @@ -110,7 +110,7 @@ void InternalCommandsHandler::loadClientStateFromStorage() { for (auto [clientId, reqId] : deserialized) { m_clientToMaxExecutedReqId[clientId] = reqId; } - LOG_INFO(GL, "raw client state: " << KVLOG(raw_json)); + LOG_DEBUG(GL, "Raw client state: " << KVLOG(raw_json)); } void InternalCommandsHandler::add(std::string &&key, @@ -165,7 +165,7 @@ void InternalCommandsHandler::execute(InternalCommandsHandler::ExecutionRequests bool isBlockAccumulationEnabled = ((requests.size() > 1) && (req.flags & bftEngine::MsgFlag::HAS_PRE_PROCESSED_FLAG)); sequenceNum = req.executionSequenceNum; - LOG_INFO(GL, KVLOG(req.clientId, req.cid, req.requestSequenceNum)); + LOG_DEBUG(GL, KVLOG(req.clientId, req.cid, req.requestSequenceNum)); res = executeWriteCommand(req.requestSize, req.request, req.executionSequenceNum, @@ -316,12 +316,12 @@ std::optional> InternalCommandsHandler::getBl } } - // Handle main key updates which should be ignored by the tests + // Handle main key updates which should be ignored by the linearizability checker of apollo tests for (auto &filteredCategory : {kConcordReconfigurationCategoryId}) { const auto concordUpdates = updates->categoryUpdates(filteredCategory); if (concordUpdates) { - const auto &u = std::get(concordUpdates->get()); - for (const auto &[key, valueWithFlags] : u.kv) { + const auto &update = std::get(concordUpdates->get()); + for (const auto &[key, valueWithFlags] : update.kv) { UNUSED(valueWithFlags); if (key[0] == concord::kvbc::keyTypes::reconfiguration_rep_main_key) { // Test categories and deployment categories are expected to be mutually exclusive @@ -384,7 +384,7 @@ std::string InternalCommandsHandler::serializeClientState() const { } serializer.to_json(json, serialized); auto serialized_raw_json = json.dump(); - LOG_INFO(GL, KVLOG(serialized_raw_json)); + LOG_DEBUG(GL, KVLOG(serialized_raw_json)); return json.dump(); } From 11dc1e57ae90d3332ecfa9accdb4d8d142c6fd6b Mon Sep 17 00:00:00 2001 From: Ynon Flum Date: Wed, 22 Mar 2023 12:28:08 +0200 Subject: [PATCH 15/18] Unrelated stability issue fix --- bftengine/src/bftengine/DbCheckpointManager.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/bftengine/src/bftengine/DbCheckpointManager.cpp b/bftengine/src/bftengine/DbCheckpointManager.cpp index 19ea02be10..5482921dc9 100644 --- a/bftengine/src/bftengine/DbCheckpointManager.cpp +++ b/bftengine/src/bftengine/DbCheckpointManager.cpp @@ -54,19 +54,15 @@ Status DbCheckpointManager::createDbCheckpoint(const CheckpointId& checkPointId, } auto end_ckpnt = Clock::now(); auto duration_chpnt_ms = std::chrono::duration_cast(end_ckpnt - start); - LOG_INFO(getLogger(), "just checkpoint duation: " << KVLOG(duration_chpnt_ms.count())); + LOG_INFO(getLogger(), "just checkpoint duration: " << KVLOG(duration_chpnt_ms.count())); prepareCheckpointCb_(lastBlockId, dbClient_->getPathForCheckpoint(checkPointId)); auto end = Clock::now(); auto duration_ms = std::chrono::duration_cast(end - start); - LOG_INFO(getLogger(), "checkpoint with cb duation: " << KVLOG(duration_ms.count())); + LOG_INFO(getLogger(), "checkpoint with cb duration: " << KVLOG(duration_ms.count())); lastDbCheckpointBlockId_.Get().Set(lastBlockId); - numOfDbCheckpointsCreated_++; auto maxSoFar = maxDbCheckpointCreationTimeMsec_.Get().Get(); maxDbCheckpointCreationTimeMsec_.Get().Set(std::max(maxSoFar, static_cast(duration_ms.count()))); - auto count = numOfDbCheckpointsCreated_.Get().Get(); - (void)count; - metrics_.UpdateAggregator(); LOG_INFO(getLogger(), "rocksdb checkpoint created: " << KVLOG(checkPointId, duration_ms.count(), seqNum)); lastCheckpointSeqNum_ = seqNum; { @@ -84,6 +80,8 @@ Status DbCheckpointManager::createDbCheckpoint(const CheckpointId& checkPointId, if (cb) cb(seqNum); } checkpointInProcessCb_(false, lastBlockId); + numOfDbCheckpointsCreated_++; + metrics_.UpdateAggregator(); } updateMetrics(); return Status::OK(); From e975270c4dab892bcb53cc0b48fa7ab350234385 Mon Sep 17 00:00:00 2001 From: Ynon Flum Date: Sun, 26 Mar 2023 16:04:11 +0300 Subject: [PATCH 16/18] Add singleSignatureScheme feature flag toggle test --- bftengine/src/bftengine/TimeServiceManager.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bftengine/src/bftengine/TimeServiceManager.hpp b/bftengine/src/bftengine/TimeServiceManager.hpp index 246c589ca3..2af799316e 100644 --- a/bftengine/src/bftengine/TimeServiceManager.hpp +++ b/bftengine/src/bftengine/TimeServiceManager.hpp @@ -48,7 +48,7 @@ class TimeServiceManager { } // Used on recovery to restore the time prior the request that is about to be re-executed. - // In order to suport a correct behaviour of the compareAndUpdate method. + // In order to support a correct behaviour of the compareAndUpdate method. void recoverTime(const ConsensusTickRep& recovered_time) { auto last_timestamp = client_.getLastTimestamp().count(); ConcordAssertLE(recovered_time, last_timestamp); From 5d9d14d0470b86afd4fea54793755e33c1b6c475 Mon Sep 17 00:00:00 2001 From: Ynon Flum Date: Tue, 28 Mar 2023 20:09:51 +0300 Subject: [PATCH 17/18] Add apollo case for tls exchange with single signature scheme --- client/bftclient/include/bftclient/config.h | 2 +- tests/apollo/test_skvbc_reconfiguration.py | 2 +- tests/apollo/util/eliot_logging.py | 3 ++- tests/simpleKVBC/TesterCRE/main.cpp | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/client/bftclient/include/bftclient/config.h b/client/bftclient/include/bftclient/config.h index d01845c368..3e00633966 100644 --- a/client/bftclient/include/bftclient/config.h +++ b/client/bftclient/include/bftclient/config.h @@ -70,7 +70,7 @@ struct ClientConfig { bool use_unified_certs = false; std::optional transaction_signing_private_key_file_path = std::nullopt; std::optional secrets_manager_config = std::nullopt; - std::optional replicas_master_key_folder_path = "./replicas_rsa_keys"; + std::optional replicas_master_key_folder_path = "./replicas_main_keys"; concord::crypto::SignatureAlgorithm message_sigs_algorithm = concord::crypto::SignatureAlgorithm::EdDSA; }; diff --git a/tests/apollo/test_skvbc_reconfiguration.py b/tests/apollo/test_skvbc_reconfiguration.py index 815519d3c3..93ae35b7b9 100644 --- a/tests/apollo/test_skvbc_reconfiguration.py +++ b/tests/apollo/test_skvbc_reconfiguration.py @@ -2214,7 +2214,7 @@ async def wait_until_cre_gets_replicas_master_keys(self, bft_network, replicas): while succ is False: succ = True for r in replicas: - master_key_path = os.path.join(bft_network.testdir, "replicas_rsa_keys", str(r), "pub_key") + master_key_path = os.path.join(bft_network.testdir, "replicas_main_keys", str(r), "pub_key") if os.path.isfile(master_key_path) is False: succ = False break diff --git a/tests/apollo/util/eliot_logging.py b/tests/apollo/util/eliot_logging.py index fc626c9f67..91598e004b 100644 --- a/tests/apollo/util/eliot_logging.py +++ b/tests/apollo/util/eliot_logging.py @@ -56,12 +56,13 @@ def set_file_destination(): latest_shortcut.unlink() latest_shortcut.symlink_to(target=Path(f'../{relative_current_run_logs}'), target_is_directory=True) + # Set the log file path + to_file(open(test_log, "a+")) test_log_file = open(test_log, "a+") to_file(test_log_file) atexit.register(lambda: test_log_file.close()) - def format_eliot_message(message): if message.get("action_status", "") == "succeeded": return diff --git a/tests/simpleKVBC/TesterCRE/main.cpp b/tests/simpleKVBC/TesterCRE/main.cpp index bbeaf6178d..6308c95f53 100644 --- a/tests/simpleKVBC/TesterCRE/main.cpp +++ b/tests/simpleKVBC/TesterCRE/main.cpp @@ -53,7 +53,7 @@ creParams setupCreParams(int argc, char** argv) { {"use-unified-certs", optional_argument, 0, 'U'}, {0, 0, 0, 0}}; creParams cre_param; - cre_param.replicasKeysFolder = "./replicas_rsa_keys"; + cre_param.replicasKeysFolder = "./replicas_main_keys"; ClientConfig& client_config = cre_param.bftConfig; int o = 0; int optionIndex = 0; From 77a5e0708c219b829d5ecf59a0628b723d4d2ad2 Mon Sep 17 00:00:00 2001 From: Ynon Flum Date: Tue, 28 Mar 2023 20:11:40 +0300 Subject: [PATCH 18/18] Redirect pruning signature generation/verification to SigManager --- Makefile | 41 +--- bftengine/include/bftengine/ReplicaConfig.hpp | 4 +- bftengine/src/bftengine/ReplicaBase.hpp | 2 +- bftengine/src/bftengine/ReplicaImp.cpp | 11 +- bftengine/src/bftengine/ReplicaLoader.cpp | 25 +- bftengine/src/bftengine/ReplicasInfo.cpp | 11 +- bftengine/src/bftengine/ReplicasInfo.hpp | 5 +- bftengine/src/bftengine/SigManager.cpp | 6 + bftengine/src/bftengine/SigManager.hpp | 8 +- kvbc/include/pruning_handler.hpp | 26 +- kvbc/src/Replica.cpp | 13 + kvbc/src/pruning_handler.cpp | 89 +++---- kvbc/test/CMakeLists.txt | 4 +- kvbc/test/pruning_test.cpp | 230 ++++++++++-------- libs/log/logger.hpp | 1 + libs/log/src/logger.cpp | 1 + tests/apollo/test_skvbc_commit_path.py | 2 +- tests/apollo/test_skvbc_view_change.py | 2 - tests/apollo/util/bft.py | 4 +- tests/apollo/util/eliot_logging.py | 5 - tests/apollo/util/pyclient/bft_client.py | 4 +- tests/simpleTest/src/simple_test_replica.cpp | 11 + 22 files changed, 249 insertions(+), 256 deletions(-) diff --git a/Makefile b/Makefile index 726e5e29ab..75ab87d0d5 100644 --- a/Makefile +++ b/Makefile @@ -49,34 +49,17 @@ else TCP_ENABLED__:=OFF endif -CONCORD_BFT_CMAKE_CXX_FLAGS_RELEASE?='-O3 -g' -CONCORD_BFT_CMAKE_USE_LOG4CPP?=ON -CONCORD_BFT_CMAKE_BUILD_UTT?=TRUE -CONCORD_BFT_CMAKE_BUILD_ROCKSDB_STORAGE?=TRUE -CONCORD_BFT_CMAKE_USE_S3_OBJECT_STORE?=TRUE -CONCORD_BFT_CMAKE_USE_OPENTRACING?=TRUE -CONCORD_BFT_CMAKE_USE_PROMETHEUS?=TRUE -CONCORD_BFT_CMAKE_USE_JAEGER?=TRUE -CONCORD_BFT_CMAKE_USE_JSON?=TRUE -CONCORD_BFT_CMAKE_USE_HTTPLIB?=TRUE -CONCORD_BFT_CMAKE_EXPORT_COMPILE_COMMANDS?=TRUE -CONCORD_BFT_CMAKE_OMIT_TEST_OUTPUT?=FALSE -CONCORD_BFT_CMAKE_KEEP_APOLLO_LOGS?=TRUE -CONCORD_BFT_CMAKE_RUN_APOLLO_TESTS?=TRUE -CONCORD_BFT_CMAKE_TRANSACTION_SIGNING_ENABLED?=TRUE -CONCORD_BFT_CMAKE_BUILD_SLOWDOWN?=FALSE -# Only useful with CONCORD_BFT_CMAKE_BUILD_TYPE:=Release -CONCORD_BFT_CMAKE_BUILD_KVBC_BENCH?=TRUE -# Only usefull with CONCORD_BFT_CMAKE_CXX_FLAGS_RELEASE=-O0 -g -CONCORD_BFT_CMAKE_ASAN?=FALSE -CONCORD_BFT_CMAKE_TSAN?=FALSE -CONCORD_BFT_CMAKE_UBSAN?=FALSE -CONCORD_BFT_CMAKE_HEAPTRACK?=FALSE -CONCORD_BFT_CMAKE_CODECOVERAGE?=FALSE -CONCORD_BFT_CMAKE_CCACHE?=TRUE -CONCORD_BFT_CMAKE_USE_FAKE_CLOCK_IN_TIME_SERVICE?=FALSE -ENABLE_RESTART_RECOVERY_TESTS?=FALSE -CONCORD_ENABLE_ALL_METRICS?=FALSE +CONCORD_BFT_CMAKE_BUILD_UTT?=ON +CONCORD_BFT_CMAKE_OMIT_TEST_OUTPUT?=OFF +CONCORD_BFT_CMAKE_KEEP_APOLLO_LOGS?=ON +CONCORD_BFT_CMAKE_RUN_APOLLO_TESTS?=ON +CONCORD_BFT_CMAKE_ASAN?=OFF +CONCORD_BFT_CMAKE_TSAN?=OFF +CONCORD_BFT_CMAKE_UBSAN?=OFF +CONCORD_BFT_CMAKE_HEAPTRACK?=OFF +CONCORD_BFT_CMAKE_CODECOVERAGE?=OFF +CONCORD_BFT_CMAKE_CCACHE?=ON +ENABLE_RESTART_RECOVERY_TESTS?=OFF # Our CMake logic won't allow more one of these flags to be raised, so having this if/else logic makes sense ifeq (${CONCORD_BFT_CMAKE_ASAN},ON) @@ -177,8 +160,6 @@ __TIMESTAMP := $(shell date +%y-%m-%d_%H-%M-%S) $(shell mkdir -p ${BUILD_DIR}) $(shell echo ${__TIMESTAMP} > ${BUILD_DIR}/timestamp) -APOLLO_TIMESTAMP := $(shell date +%y-%m-%d_%H-%M-%S) - BASIC_RUN_PARAMS?=-it --init --rm --privileged=true \ --memory-swap -1 \ --cap-add NET_ADMIN --cap-add=SYS_PTRACE --ulimit core=-1 \ diff --git a/bftengine/include/bftengine/ReplicaConfig.hpp b/bftengine/include/bftengine/ReplicaConfig.hpp index 2cd6af5ce2..a875d4ef01 100644 --- a/bftengine/include/bftengine/ReplicaConfig.hpp +++ b/bftengine/include/bftengine/ReplicaConfig.hpp @@ -306,11 +306,11 @@ class ReplicaConfig : public concord::serialize::SerializableFactory(it->second); return defaultValue; } - inline std::set>>* getPublicKeysOfClients() { + inline const std::set>>* getPublicKeysOfClients() const { return (clientTransactionSigningEnabled || !clientsKeysPrefix.empty()) ? &publicKeysOfClients : nullptr; } - std::string getOperatorPublicKey() { + std::string getOperatorPublicKey() const { std::ifstream op_key_file(pathToOperatorPublicKey_); if (!op_key_file.fail()) { std::stringstream buffer; diff --git a/bftengine/src/bftengine/ReplicaBase.hpp b/bftengine/src/bftengine/ReplicaBase.hpp index aa625a4ea3..3096ae15fe 100644 --- a/bftengine/src/bftengine/ReplicaBase.hpp +++ b/bftengine/src/bftengine/ReplicaBase.hpp @@ -112,7 +112,7 @@ class ReplicaBase { static const uint16_t ALL_OTHER_REPLICAS = UINT16_MAX; const ReplicaConfig& config_; - ReplicasInfo* repsInfo = nullptr; + const ReplicasInfo* repsInfo = nullptr; std::shared_ptr msgsCommunicator_; std::shared_ptr msgHandlers_; std::shared_ptr bftRequestsHandler_; diff --git a/bftengine/src/bftengine/ReplicaImp.cpp b/bftengine/src/bftengine/ReplicaImp.cpp index 1ecd2fd8cb..272144d193 100644 --- a/bftengine/src/bftengine/ReplicaImp.cpp +++ b/bftengine/src/bftengine/ReplicaImp.cpp @@ -4434,16 +4434,7 @@ ReplicaImp::ReplicaImp(bool firstTime, if (firstTime) { repsInfo = new ReplicasInfo(config_, dynamicCollectorForPartialProofs, dynamicCollectorForExecutionProofs); - sigManager_ = SigManager::init(config_.replicaId, - config_.replicaPrivateKey, - config_.publicKeysOfReplicas, - concord::crypto::KeyFormat::HexaDecimalStrippedFormat, - ReplicaConfig::instance().getPublicKeysOfClients(), - concord::crypto::KeyFormat::PemFormat, - {{repsInfo->getIdOfOperator(), - ReplicaConfig::instance().getOperatorPublicKey(), - concord::crypto::KeyFormat::PemFormat}}, - *repsInfo); + sigManager_ = SigManager::owningInstance(); viewsManager = new ViewsManager(repsInfo); } else { repsInfo = replicasInfo; diff --git a/bftengine/src/bftengine/ReplicaLoader.cpp b/bftengine/src/bftengine/ReplicaLoader.cpp index 8e60e23319..c7dffd11c7 100644 --- a/bftengine/src/bftengine/ReplicaLoader.cpp +++ b/bftengine/src/bftengine/ReplicaLoader.cpp @@ -47,25 +47,14 @@ namespace impl { namespace { ReplicaLoader::ErrorCode loadConfig(LoadedReplicaData &ld) { - ld.repsInfo = new ReplicasInfo(ld.repConfig, dynamicCollectorForPartialProofs, dynamicCollectorForExecutionProofs); auto &config = ld.repConfig; - ld.sigManager = SigManager::init(config.replicaId, - config.replicaPrivateKey, - config.publicKeysOfReplicas, - concord::crypto::KeyFormat::HexaDecimalStrippedFormat, - ReplicaConfig::instance().getPublicKeysOfClients(), - concord::crypto::KeyFormat::PemFormat, - {{ld.repsInfo->getIdOfOperator(), - ReplicaConfig::instance().getOperatorPublicKey(), - concord::crypto::KeyFormat::PemFormat}}, - *ld.repsInfo); - - std::unique_ptr cryptoSys = std::make_unique(ld.repConfig.thresholdSystemType_, - ld.repConfig.thresholdSystemSubType_, - ld.repConfig.numReplicas, - ld.repConfig.numReplicas); - cryptoSys->loadKeys(ld.repConfig.thresholdPublicKey_, ld.repConfig.thresholdVerificationKeys_); - cryptoSys->loadPrivateKey(ld.repConfig.replicaId, ld.repConfig.thresholdPrivateKey_); + ld.repsInfo = new ReplicasInfo(config, dynamicCollectorForPartialProofs, dynamicCollectorForExecutionProofs); + ld.sigManager = SigManager::owningInstance(); + + std::unique_ptr cryptoSys = std::make_unique( + config.thresholdSystemType_, config.thresholdSystemSubType_, config.numReplicas, config.numReplicas); + cryptoSys->loadKeys(config.thresholdPublicKey_, config.thresholdVerificationKeys_); + cryptoSys->loadPrivateKey(config.replicaId, config.thresholdPrivateKey_); bftEngine::CryptoManager::init(std::move(cryptoSys)); return Succ; diff --git a/bftengine/src/bftengine/ReplicasInfo.cpp b/bftengine/src/bftengine/ReplicasInfo.cpp index 13d3060b57..b3c570c5c4 100644 --- a/bftengine/src/bftengine/ReplicasInfo.cpp +++ b/bftengine/src/bftengine/ReplicasInfo.cpp @@ -146,11 +146,12 @@ ReplicasInfo::ReplicasInfo(const ReplicaConfig& config, } if (start != end) LOG_INFO(GL, "Principal ids in _idsOfInternalClients: " << start << " to " << end - 1); return ret; - }()} { - _operator_id = config.operatorEnabled_ - ? static_cast(config.numReplicas + config.numRoReplicas + config.numOfClientProxies + - config.numOfExternalClients + config.numOfClientServices - 1) - : 0; + }()}, + + _operator_id{static_cast( + config.operatorEnabled_ ? config.numReplicas + config.numRoReplicas + config.numOfClientProxies + + config.numOfExternalClients + config.numOfClientServices - 1 + : 0)} { ConcordAssert(_numberOfReplicas == (3 * _fVal + 2 * _cVal + 1)); } diff --git a/bftengine/src/bftengine/ReplicasInfo.hpp b/bftengine/src/bftengine/ReplicasInfo.hpp index b6a65efeaa..81b3a0994a 100644 --- a/bftengine/src/bftengine/ReplicasInfo.hpp +++ b/bftengine/src/bftengine/ReplicasInfo.hpp @@ -20,6 +20,9 @@ class ReplicaConfig; namespace impl { +/** + * An immutable class holding the the ids of all the participants in the network + */ class ReplicasInfo { public: ReplicasInfo(const ReplicaConfig&, bool dynamicCollectorForPartialProofs, bool dynamicCollectorForExecutionProofs); @@ -110,7 +113,7 @@ class ReplicasInfo { const std::set _idsOfInternalClients; // Currently we support only a single operator entity in the system - PrincipalId _operator_id; + const PrincipalId _operator_id = 0; }; } // namespace impl } // namespace bftEngine diff --git a/bftengine/src/bftengine/SigManager.cpp b/bftengine/src/bftengine/SigManager.cpp index 14ed5df1e8..a4df8073c5 100644 --- a/bftengine/src/bftengine/SigManager.cpp +++ b/bftengine/src/bftengine/SigManager.cpp @@ -47,6 +47,11 @@ SigManager* SigManager::instance() { return s_sm.get(); } +std::shared_ptr SigManager::owningInstance() { + ConcordAssertNE(s_sm.get(), nullptr); + return s_sm; +} + void SigManager::reset(std::shared_ptr other) { s_sm = other; } std::shared_ptr SigManager::init( @@ -457,6 +462,7 @@ SeqNum SigManager::getReplicaLastExecutedSeq() const { ConcordAssert(replicasInfo_.isIdOfReplica(myId_) || replicasInfo_.isRoReplica()); return replicaLastExecutedSeq_; } +const ReplicasInfo& SigManager::getReplicasInfo() const { return replicasInfo_; } } // namespace impl } // namespace bftEngine diff --git a/bftengine/src/bftengine/SigManager.hpp b/bftengine/src/bftengine/SigManager.hpp index 63bb5f307e..57f3a51a62 100644 --- a/bftengine/src/bftengine/SigManager.hpp +++ b/bftengine/src/bftengine/SigManager.hpp @@ -19,6 +19,7 @@ #include "crypto/signer.hpp" #include "crypto/verifier.hpp" #include "SysConsts.hpp" +#include "ReplicasInfo.hpp" #include #include #include @@ -35,8 +36,6 @@ class IThresholdVerifier; namespace bftEngine { namespace impl { -class ReplicasInfo; - class SigManager { public: using Key = std::string; @@ -44,6 +43,7 @@ class SigManager { virtual ~SigManager() = default; static SigManager* instance(); + static std::shared_ptr owningInstance(); static void reset(std::shared_ptr other); // It is the caller responsibility to deallocate (delete) the object @@ -129,6 +129,8 @@ class SigManager { const concord::crypto::IVerifier& extractVerifierFromMultisig(std::shared_ptr thresholdVerifier, PrincipalId id) const; + const ReplicasInfo& getReplicasInfo() const; + protected: static constexpr uint16_t updateMetricsAggregatorThresh = 1000; @@ -153,7 +155,7 @@ class SigManager { std::unique_ptr mySigner_; std::map> verifiers_; bool clientTransactionSigningEnabled_ = true; - const ReplicasInfo& replicasInfo_; + const ReplicasInfo replicasInfo_; // The ownership model of a SigManager object depends on its use static std::shared_ptr s_sm; diff --git a/kvbc/include/pruning_handler.hpp b/kvbc/include/pruning_handler.hpp index 2f3a6237a8..5c02fc6f06 100644 --- a/kvbc/include/pruning_handler.hpp +++ b/kvbc/include/pruning_handler.hpp @@ -32,7 +32,7 @@ class PruningSigner { public: // Construct by passing the configuration for the node the signer is running // on. - PruningSigner(const std::string &key); + PruningSigner(); // Sign() methods sign the passed message and store the signature in the // 'signature' field of the message. An exception is thrown on error. // @@ -41,9 +41,6 @@ class PruningSigner { // application-level signature rather than a Concord-BFT Principal's RSA/EdDSA // signature. void sign(concord::messages::LatestPrunableBlock &); - - private: - std::unique_ptr signer_; }; // This class verifies pruning messages that were signed by serializing message @@ -55,7 +52,7 @@ class PruningSigner { class PruningVerifier { public: // Construct by passing the system configuration. - PruningVerifier(const std::set> &replicasPublicKeys); + PruningVerifier(); // Verify() methods verify that the message comes from the advertised sender. // Methods return true on successful verification and false on unsuccessful. // An exception is thrown on error. @@ -76,22 +73,6 @@ class PruningVerifier { }; bool verify(std::uint64_t sender, const std::string &ser, const std::string &signature) const; - - using ReplicaVector = std::vector; - - // Get a replica from the replicas vector by its index. - const Replica &getReplica(ReplicaVector::size_type idx) const; - - // A vector of all the replicas in the system. - ReplicaVector replicas_; - // We map a principal_id to a replica index in the replicas_ vector to be able - // to verify a message through the Replica's verifier that is associated with - // its public key. - std::unordered_map principal_to_replica_idx_; - - // Contains a set of replica principal_ids for use in verification. Filled in - // once during construction. - std::unordered_set replica_ids_; }; class PruningHandler : public concord::reconfiguration::OperatorCommandsReconfigurationHandler { // This class implements the KVB pruning state machine. Main functionalities @@ -171,7 +152,6 @@ class PruningHandler : public concord::reconfiguration::OperatorCommandsReconfig // Throws on errors. void pruneThroughBlockId(kvbc::BlockId block_id) const; uint64_t getBlockBftSequenceNumber(kvbc::BlockId) const; - logging::Logger logger_; PruningSigner signer_; PruningVerifier verifier_; kvbc::IReader &ro_storage_; @@ -198,7 +178,7 @@ class ReadOnlyReplicaPruningHandler : public concord::reconfiguration::OperatorC IReader &ro_storage) : concord::reconfiguration::OperatorCommandsReconfigurationHandler{operator_pkey_path, type}, ro_storage_{ro_storage}, - signer_{bftEngine::ReplicaConfig::instance().replicaPrivateKey}, + signer_{}, pruning_enabled_{bftEngine::ReplicaConfig::instance().pruningEnabled_}, replica_id_{bftEngine::ReplicaConfig::instance().replicaId} {} bool handle(const concord::messages::LatestPrunableBlockRequest &, diff --git a/kvbc/src/Replica.cpp b/kvbc/src/Replica.cpp index 2dff5b6df8..7e37525898 100644 --- a/kvbc/src/Replica.cpp +++ b/kvbc/src/Replica.cpp @@ -316,8 +316,21 @@ void Replica::saveReconfigurationCmdToResPages(const std::string &key) { void Replica::createReplicaAndSyncState() { ConcordAssertNE(m_kvBlockchain, nullptr); + + ReplicasInfo repsInfo{replicaConfig_, dynamicCollectorForPartialProofs, dynamicCollectorForExecutionProofs}; + SigManager::init( + replicaConfig_.replicaId, + replicaConfig_.replicaPrivateKey, + replicaConfig_.publicKeysOfReplicas, + concord::crypto::KeyFormat::HexaDecimalStrippedFormat, + replicaConfig_.getPublicKeysOfClients(), + concord::crypto::KeyFormat::PemFormat, + {{repsInfo.getIdOfOperator(), replicaConfig_.getOperatorPublicKey(), concord::crypto::KeyFormat::PemFormat}}, + repsInfo); + auto requestHandler = KvbcRequestHandler::create(m_cmdHandler, cronTableRegistry_, *m_kvBlockchain, aggregator_); registerReconfigurationHandlers(requestHandler); + m_replicaPtr = bftEngine::ReplicaFactory::createReplica( replicaConfig_, requestHandler, m_stateTransfer, m_ptrComm.get(), m_metadataStorage, pm_, secretsManager_); requestHandler->setPersistentStorage(m_replicaPtr->persistentStorage()); diff --git a/kvbc/src/pruning_handler.cpp b/kvbc/src/pruning_handler.cpp index 6f546076fe..faa35cc7bb 100644 --- a/kvbc/src/pruning_handler.cpp +++ b/kvbc/src/pruning_handler.cpp @@ -16,45 +16,32 @@ #include "pruning_handler.hpp" #include "categorization/versioned_kv_category.h" #include "kvbc_key_types.hpp" +#include "SigManager.hpp" namespace concord::kvbc::pruning { using bftEngine::ReplicaConfig; void PruningSigner::sign(concord::messages::LatestPrunableBlock& block) { + auto& sigManager = *bftEngine::impl::SigManager::instance(); std::ostringstream oss; std::string ser; oss << block.replica << block.block_id; ser = oss.str(); - std::vector signature(signer_->signatureLength()); - auto actualSignatureLength = signer_->sign(ser, signature.data()); - ConcordAssertEQ(actualSignatureLength, signer_->signatureLength()); + std::vector signature(sigManager.getMySigLength()); + sigManager.sign( + sigManager.getReplicaLastExecutedSeq(), ser.data(), ser.size(), reinterpret_cast(signature.data())); + block.signature = std::move(signature); } -PruningSigner::PruningSigner(const std::string& key) - : signer_{concord::crypto::Factory::getSigner(key, ReplicaConfig::instance().replicaMsgSigningAlgo)} {} - -PruningVerifier::PruningVerifier(const std::set>& replicasPublicKeys) { - auto i = 0u; - for (auto& [idx, pkey] : replicasPublicKeys) { - replicas_.push_back( - Replica{idx, concord::crypto::Factory::getVerifier(pkey, ReplicaConfig::instance().replicaMsgSigningAlgo)}); - const auto ins_res = replica_ids_.insert(replicas_.back().principal_id); - if (!ins_res.second) { - throw std::runtime_error{"PruningVerifier found duplicate replica principal_id: " + - std::to_string(replicas_.back().principal_id)}; - } +PruningSigner::PruningSigner() {} - const auto& replica = replicas_.back(); - principal_to_replica_idx_[replica.principal_id] = i; - i++; - } -} +PruningVerifier::PruningVerifier() {} bool PruningVerifier::verify(const concord::messages::LatestPrunableBlock& block) const { // LatestPrunableBlock can only be sent by replicas and not by client proxies. - if (replica_ids_.find(block.replica) == std::end(replica_ids_)) { + if (!SigManager::instance()->hasVerifier(block.replica)) { return false; } std::ostringstream oss; @@ -66,46 +53,51 @@ bool PruningVerifier::verify(const concord::messages::LatestPrunableBlock& block } bool PruningVerifier::verify(const concord::messages::PruneRequest& request) const { - if (request.latest_prunable_block.size() != static_cast(replica_ids_.size())) { + const auto& repInfo = SigManager::instance()->getReplicasInfo(); + const auto expected_response_count = repInfo.getNumberOfReplicas() + repInfo.getNumberOfRoReplicas(); + if (request.latest_prunable_block.size() != static_cast(expected_response_count)) { + LOG_ERROR( + PRUNING_LOG, + "Invalid number of latest prunable block responses in prune request" << KVLOG( + request.latest_prunable_block.size(), repInfo.getNumberOfReplicas(), repInfo.getNumberOfRoReplicas())); return false; } // PruneRequest can only be sent by client proxies and not by replicas. - if (replica_ids_.find(request.sender) != std::end(replica_ids_)) { + if (repInfo.isIdOfReplica(request.sender)) { + LOG_ERROR(PRUNING_LOG, "Invalid sender of prune request, sender id:" << request.sender); return false; } // Note PruningVerifier does not handle verification of the operator's // signature authorizing this pruning order, as the operator's signature is a // dedicated application-level signature rather than one of the Concord-BFT - // principals' RSA signatures. + // principals' main signatures. // Verify that *all* replicas have responded with valid responses. - auto replica_ids_to_verify = replica_ids_; for (auto& block : request.latest_prunable_block) { + auto currentId = block.replica; if (!verify(block)) { + LOG_ERROR(PRUNING_LOG, "Failed to verify latest prunable block of replica id:" << currentId); return false; } - auto it = replica_ids_to_verify.find(block.replica); - if (it == std::end(replica_ids_to_verify)) { + + if (!(repInfo.isIdOfReplica(currentId) || repInfo.isIdOfPeerRoReplica(currentId))) { + LOG_ERROR(PRUNING_LOG, "latest prunable block is not of a replica nor an RO replica:" << currentId); return false; } - replica_ids_to_verify.erase(it); } - return replica_ids_to_verify.empty(); + + return true; } bool PruningVerifier::verify(std::uint64_t sender, const std::string& ser, const std::string& signature) const { - auto it = principal_to_replica_idx_.find(sender); - if (it == std::cend(principal_to_replica_idx_)) { + if (!SigManager::instance()->hasVerifier(sender)) { return false; } - return getReplica(it->second).verifier->verify(ser, signature); -} - -const PruningVerifier::Replica& PruningVerifier::getReplica(ReplicaVector::size_type idx) const { - return replicas_[idx]; + auto& sigManager = *bftEngine::impl::SigManager::instance(); + return sigManager.verifySig(sender, ser, signature); } PruningHandler::PruningHandler(const std::string& operator_pkey_path, @@ -115,9 +107,8 @@ PruningHandler::PruningHandler(const std::string& operator_pkey_path, kvbc::IBlocksDeleter& blocks_deleter, bool run_async) : concord::reconfiguration::OperatorCommandsReconfigurationHandler{operator_pkey_path, type}, - logger_{logging::getLogger("concord.pruning")}, - signer_{bftEngine::ReplicaConfig::instance().replicaPrivateKey}, - verifier_{bftEngine::ReplicaConfig::instance().publicKeysOfReplicas}, + signer_{}, + verifier_{}, ro_storage_{ro_storage}, blocks_adder_{blocks_adder}, blocks_deleter_{blocks_deleter}, @@ -173,7 +164,7 @@ bool PruningHandler::handle(const concord::messages::PruneRequest& request, "LatestPrunableBlock messages did not bear correct signatures " "from the claimed replicas."; concord::messages::ReconfigurationErrorMsg error_msg; - LOG_WARN(logger_, error); + LOG_WARN(PRUNING_LOG, error); error_msg.error_msg = error; rres.response = error_msg; return false; @@ -214,10 +205,10 @@ void PruningHandler::pruneThroughBlockId(kvbc::BlockId block_id) const { try { blocks_deleter_.deleteBlocksUntil(until, false); } catch (std::exception& e) { - LOG_FATAL(logger_, e.what()); + LOG_FATAL(PRUNING_LOG, e.what()); std::terminate(); } catch (...) { - LOG_FATAL(logger_, "Error while running pruning"); + LOG_FATAL(PRUNING_LOG, "Error while running pruning"); std::terminate(); } // We grab a mutex to handle the case in which we ask for pruning status @@ -226,11 +217,11 @@ void PruningHandler::pruneThroughBlockId(kvbc::BlockId block_id) const { bftEngine::ControlStateManager::instance().setPruningProcess(false); }; if (run_async_) { - LOG_INFO(logger_, "running pruning in async mode"); + LOG_INFO(PRUNING_LOG, "running pruning in async mode"); async_pruning_res_ = std::async(prune, block_id + 1); (void)async_pruning_res_; } else { - LOG_INFO(logger_, "running pruning in sync mode"); + LOG_INFO(PRUNING_LOG, "running pruning in sync mode"); prune(block_id + 1); } } @@ -251,7 +242,7 @@ bool PruningHandler::handle(const concord::messages::PruneStatusRequest&, prune_status.last_pruned_block = (genesis_id > INITIAL_GENESIS_BLOCK_ID ? genesis_id - 1 : 0); prune_status.in_progress = bftEngine::ControlStateManager::instance().getPruningProcessStatus(); rres.response = prune_status; - LOG_INFO(logger_, "Pruning status is " << KVLOG(prune_status.in_progress)); + LOG_INFO(PRUNING_LOG, "Pruning status is " << KVLOG(prune_status.in_progress)); return true; } @@ -260,16 +251,16 @@ uint64_t PruningHandler::getBlockBftSequenceNumber(kvbc::BlockId bid) const { concord::kvbc::categorization::kConcordInternalCategoryId, std::string{kvbc::keyTypes::bft_seq_num_key}, bid); uint64_t sequenceNum = 0; if (!opt_value) { - LOG_WARN(logger_, "Unable to get block"); + LOG_WARN(PRUNING_LOG, "Unable to get block"); return sequenceNum; } auto value = std::get(*opt_value); if (value.data.empty()) { - LOG_WARN(logger_, "value has zero-length"); + LOG_WARN(PRUNING_LOG, "value has zero-length"); return sequenceNum; } sequenceNum = concordUtils::fromBigEndianBuffer(value.data.data()); - LOG_DEBUG(logger_, "sequenceNum = " << sequenceNum); + LOG_DEBUG(PRUNING_LOG, "sequenceNum = " << sequenceNum); return sequenceNum; } } // namespace concord::kvbc::pruning diff --git a/kvbc/test/CMakeLists.txt b/kvbc/test/CMakeLists.txt index 9485068cda..6395aa7ca3 100644 --- a/kvbc/test/CMakeLists.txt +++ b/kvbc/test/CMakeLists.txt @@ -306,8 +306,8 @@ if (BUILD_ROCKSDB_STORAGE) stdc++fs ) - - add_executable(pruning_test pruning_test.cpp) + add_executable(pruning_test pruning_test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../bftengine/tests/messages/helper.cpp) + target_include_directories(pruning_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../bftengine/tests/messages) add_test(pruning_test pruning_test) target_link_libraries(pruning_test GTest::GTest diff --git a/kvbc/test/pruning_test.cpp b/kvbc/test/pruning_test.cpp index 1a86fc58c3..0f701ff110 100644 --- a/kvbc/test/pruning_test.cpp +++ b/kvbc/test/pruning_test.cpp @@ -18,7 +18,9 @@ #include "db_interfaces.h" #include "util/endianness.hpp" #include "pruning_handler.hpp" - +#include "SigManager.hpp" +#include "CryptoManager.hpp" +#include "helper.hpp" #include "storage/test/storage_test_common.h" #include @@ -36,40 +38,23 @@ using concord::kvbc::BlockId; using namespace concord::kvbc; using namespace concord::kvbc::categorization; using namespace concord::kvbc::pruning; -using concord::crypto::SignatureAlgorithm; using bftEngine::ReplicaConfig; -using concord::crypto::generateEdDSAKeyPair; +using bftEngine::impl::SigManager; +using bftEngine::CryptoManager; namespace { -const NodeIdType replica_0 = 0; -const NodeIdType replica_1 = 1; -const NodeIdType replica_2 = 2; -const NodeIdType replica_3 = 3; -constexpr uint8_t noOfReplicas = 4U; - -std::pair keyPair[noOfReplicas]; -bftEngine::ReplicaConfig &replicaConfig = bftEngine::ReplicaConfig::instance(); -std::map private_keys_of_replicas; - -void setUpKeysConfiguration_4() { - for (auto i = 0; i < noOfReplicas; ++i) { - if (ReplicaConfig::instance().replicaMsgSigningAlgo == SignatureAlgorithm::EdDSA) { - keyPair[i] = generateEdDSAKeyPair(); - } - } - replicaConfig.publicKeysOfReplicas.insert(std::pair(replica_0, keyPair[0].second)); - replicaConfig.publicKeysOfReplicas.insert(std::pair(replica_1, keyPair[1].second)); - replicaConfig.publicKeysOfReplicas.insert(std::pair(replica_2, keyPair[2].second)); - replicaConfig.publicKeysOfReplicas.insert(std::pair(replica_3, keyPair[3].second)); - private_keys_of_replicas[replica_0] = keyPair[0].first; - private_keys_of_replicas[replica_1] = keyPair[1].first; - private_keys_of_replicas[replica_2] = keyPair[2].first; - private_keys_of_replicas[replica_3] = keyPair[3].first; -} +const auto GENESIS_BLOCK_ID = BlockId{1}; +const auto LAST_BLOCK_ID = BlockId{150}; +const auto REPLICA_PRINCIPAL_ID_START = 0; +const auto CLIENT_PRINCIPAL_ID_START = 20000; class test_rocksdb : public ::testing::Test { + static constexpr const uint8_t noOfReplicas = 4U; + void SetUp() override { + GL.setLogLevel(logging::DEBUG_LOG_LEVEL); + setUpKeysConfiguration_4(); destroyDb(); db = TestRocksDb::createNative(); } @@ -83,13 +68,86 @@ class test_rocksdb : public ::testing::Test { } protected: + ReplicaConfig &replicaConfig = ReplicaConfig::instance(); + std::map private_keys_of_replicas; + std::array, noOfReplicas> replicasInfo; + std::array, noOfReplicas> sigManagers; + std::array, noOfReplicas> cryptoManagers; + static const std::pair s_eddsaKeyPairs[noOfReplicas]; + + void setUpKeysConfiguration_4() { + replicaConfig.replicaId = 0; + replicaConfig.numReplicas = 4; + replicaConfig.fVal = 1; + replicaConfig.cVal = 0; + replicaConfig.numOfClientProxies = 4; + replicaConfig.numOfExternalClients = 15; + + std::vector hexPublicKeys(noOfReplicas); + replicaConfig.publicKeysOfReplicas.clear(); + for (auto i = 0; i < noOfReplicas; ++i) { + hexPublicKeys[i] = s_eddsaKeyPairs[i].second; + private_keys_of_replicas.emplace(i, s_eddsaKeyPairs[i].first); + replicaConfig.publicKeysOfReplicas.emplace(i, hexPublicKeys[i]); + } + + for (auto i = 0; i < noOfReplicas; i++) { + replicasInfo[i] = std::make_unique(replicaConfig, true, true); + sigManagers[i] = SigManager::init(i, + private_keys_of_replicas.at(i), + replicaConfig.publicKeysOfReplicas, + concord::crypto::KeyFormat::HexaDecimalStrippedFormat, + nullptr, + concord::crypto::KeyFormat::HexaDecimalStrippedFormat, + {}, + *replicasInfo[i].get()); + cryptoManagers[i] = CryptoManager::init( + std::make_unique(i, hexPublicKeys, private_keys_of_replicas.at(i))); + } + } + + void switchReplicaContext(NodeIdType id) { + LOG_INFO(GL, "Switching replica context to replica " << id); + SigManager::reset(sigManagers[id]); + CryptoManager::reset(cryptoManagers[id]); + } + + void signAs(NodeIdType id, concord::messages::LatestPrunableBlock &block) { + switchReplicaContext(id); + PruningSigner{}.sign(block); + } + + concord::messages::PruneRequest ConstructPruneRequest(std::size_t client_idx, + BlockId min_prunable_block_id = LAST_BLOCK_ID) { + concord::messages::PruneRequest prune_req; + prune_req.sender = client_idx; + + for (uint64_t i = 0u; i < noOfReplicas; i++) { + auto &latest_block = prune_req.latest_prunable_block.emplace_back(concord::messages::LatestPrunableBlock()); + latest_block.replica = i; + // Send different block IDs. + latest_block.block_id = min_prunable_block_id + i; + + signAs(i, latest_block); + } + return prune_req; + } + std::shared_ptr db; }; -const auto GENESIS_BLOCK_ID = BlockId{1}; -const auto LAST_BLOCK_ID = BlockId{150}; -const auto REPLICA_PRINCIPAL_ID_START = 0; -const auto CLIENT_PRINCIPAL_ID_START = 20000; +// clang-format off +const std::pair test_rocksdb::s_eddsaKeyPairs[noOfReplicas] = { + {"61498efe1764b89357a02e2887d224154006ceacf26269f8695a4af561453eef", + "386f4fb049a5d8bb0706d3793096c8f91842ce380dfc342a2001d50dfbc901f4"}, + {"247a74ab3620ec6b9f5feab9ee1f86521da3fa2804ad45bb5bf2c5b21ef105bc", + "3f9e7dbde90477c24c1bacf14e073a356c1eca482d352d9cc0b16560a4e7e469"}, + {"fb539bc3d66deda55524d903da26dbec1f4b6abf41ec5db521e617c64eb2c341", + "2311c6013ff657844669d8b803b2e1ed33fe06eed445f966a800a8fbb8d790e8"}, + {"55ea66e855b83ec4a02bd8fcce6bb4426ad3db2a842fa2a2a6777f13e40a4717", + "1ba7449655784fc9ce193a7887de1e4d3d35f7c82b802440c4f28bf678a34b34"}, +}; +// clang-format on class TestStorage : public IReader, public IBlockAdder, public IBlocksDeleter { public: @@ -245,42 +303,19 @@ void InitBlockchainStorage(std::size_t replica_count, ASSERT_EQ(s.getLastBlockId(), LAST_BLOCK_ID); } -concord::messages::PruneRequest ConstructPruneRequest(std::size_t client_idx, - const std::map &private_keys, - BlockId min_prunable_block_id = LAST_BLOCK_ID) { - concord::messages::PruneRequest prune_req; - prune_req.sender = client_idx; - uint64_t i = 0u; - for (auto &[idx, pkey] : private_keys) { - auto &latest_block = prune_req.latest_prunable_block.emplace_back(concord::messages::LatestPrunableBlock()); - latest_block.replica = idx; - // Send different block IDs. - latest_block.block_id = min_prunable_block_id + i; - - auto block_signer = PruningSigner{pkey}; - block_signer.sign(latest_block); - i++; - } - return prune_req; -} - TEST_F(test_rocksdb, sign_verify_correct) { const auto replica_count = 4; uint64_t sending_id = 0; uint64_t client_proxy_count = 4; - const auto verifier = PruningVerifier{replicaConfig.publicKeysOfReplicas}; - std::vector signers; - signers.reserve(replica_count); - for (auto i = 0; i < replica_count; ++i) { - signers.emplace_back(PruningSigner{private_keys_of_replicas[i]}); - } + const auto verifier = PruningVerifier{}; // Sign and verify a LatestPrunableBlock message. { concord::messages::LatestPrunableBlock block; block.replica = REPLICA_PRINCIPAL_ID_START + sending_id; block.block_id = LAST_BLOCK_ID; - signers[sending_id].sign(block); + + signAs(sending_id, block); ASSERT_TRUE(verifier.verify(block)); } @@ -293,7 +328,7 @@ TEST_F(test_rocksdb, sign_verify_correct) { auto &block = request.latest_prunable_block.emplace_back(concord::messages::LatestPrunableBlock()); block.replica = REPLICA_PRINCIPAL_ID_START + i; block.block_id = LAST_BLOCK_ID; - signers[i].sign(block); + signAs(i, block); } ASSERT_TRUE(verifier.verify(request)); } @@ -303,19 +338,14 @@ TEST_F(test_rocksdb, verify_malformed_messages) { const auto replica_count = 4; const auto client_proxy_count = replica_count; const auto sending_id = 1; - const auto verifier = PruningVerifier{replicaConfig.publicKeysOfReplicas}; - std::vector signers; - signers.reserve(replica_count); - for (auto i = 0; i < replica_count; ++i) { - signers.emplace_back(PruningSigner{private_keys_of_replicas[i]}); - } + const auto verifier = PruningVerifier{}; // Break verification of LatestPrunableBlock messages. { concord::messages::LatestPrunableBlock block; block.replica = REPLICA_PRINCIPAL_ID_START + sending_id; block.block_id = LAST_BLOCK_ID; - signers[sending_id].sign(block); + signAs(sending_id, block); // Change the replica ID after signing. block.replica = REPLICA_PRINCIPAL_ID_START + sending_id + 1; @@ -346,7 +376,7 @@ TEST_F(test_rocksdb, verify_malformed_messages) { auto &block = request.latest_prunable_block.emplace_back(concord::messages::LatestPrunableBlock()); block.replica = REPLICA_PRINCIPAL_ID_START + i; block.block_id = LAST_BLOCK_ID; - signers[i].sign(block); + signAs(i, block); } request.sender = request.sender + 1; @@ -362,7 +392,7 @@ TEST_F(test_rocksdb, verify_malformed_messages) { auto &block = request.latest_prunable_block.emplace_back(concord::messages::LatestPrunableBlock()); block.replica = REPLICA_PRINCIPAL_ID_START + i; block.block_id = LAST_BLOCK_ID; - signers[i].sign(block); + signAs(i, block); } ASSERT_FALSE(verifier.verify(request)); @@ -376,7 +406,7 @@ TEST_F(test_rocksdb, verify_malformed_messages) { auto &block = request.latest_prunable_block.emplace_back(concord::messages::LatestPrunableBlock()); block.replica = REPLICA_PRINCIPAL_ID_START + i; block.block_id = LAST_BLOCK_ID; - signers[i].sign(block); + signAs(i, block); } request.latest_prunable_block[0].replica = REPLICA_PRINCIPAL_ID_START + replica_count + 8; @@ -393,14 +423,14 @@ TEST_F(test_rocksdb, sm_latest_prunable_request_correct_num_bocks_to_keep) { replicaConfig.pruningEnabled_ = true; TestStorage storage(db); auto &blocks_deleter = storage; - const auto verifier = PruningVerifier{replicaConfig.publicKeysOfReplicas}; - replicaConfig.replicaPrivateKey = keyPair[1].first; + const auto verifier = PruningVerifier{}; + switchReplicaContext(replica_idx); InitBlockchainStorage(replica_count, storage); // Construct the pruning state machine with a nullptr TimeContract to verify // it works in case the time service is disabled. - auto sm = PruningHandler{bftEngine::ReplicaConfig::instance().pathToOperatorPublicKey_, - bftEngine::ReplicaConfig::instance().operatorMsgSigningAlgo, + auto sm = PruningHandler{ReplicaConfig::instance().pathToOperatorPublicKey_, + ReplicaConfig::instance().operatorMsgSigningAlgo, storage, storage, blocks_deleter, @@ -420,13 +450,14 @@ TEST_F(test_rocksdb, sm_latest_prunable_request_big_num_blocks_to_keep) { replicaConfig.numBlocksToKeep_ = num_blocks_to_keep; replicaConfig.replicaId = 1; replicaConfig.pruningEnabled_ = true; - replicaConfig.replicaPrivateKey = keyPair[1].first; + replicaConfig.replicaPrivateKey = s_eddsaKeyPairs[1].first; + switchReplicaContext(1); TestStorage storage(db); auto &blocks_deleter = storage; - const auto verifier = PruningVerifier{replicaConfig.publicKeysOfReplicas}; + const auto verifier = PruningVerifier{}; - auto sm = PruningHandler{bftEngine::ReplicaConfig::instance().pathToOperatorPublicKey_, - bftEngine::ReplicaConfig::instance().operatorMsgSigningAlgo, + auto sm = PruningHandler{ReplicaConfig::instance().pathToOperatorPublicKey_, + ReplicaConfig::instance().operatorMsgSigningAlgo, storage, storage, blocks_deleter, @@ -450,15 +481,16 @@ TEST_F(test_rocksdb, sm_latest_prunable_request_no_pruning_conf) { replicaConfig.numBlocksToKeep_ = num_blocks_to_keep; replicaConfig.replicaId = 1; replicaConfig.pruningEnabled_ = true; - replicaConfig.replicaPrivateKey = keyPair[1].first; + replicaConfig.replicaPrivateKey = s_eddsaKeyPairs[1].first; + switchReplicaContext(1); TestStorage storage(db); auto &blocks_deleter = storage; - const auto verifier = PruningVerifier{replicaConfig.publicKeysOfReplicas}; + const auto verifier = PruningVerifier{}; InitBlockchainStorage(replica_count, storage); - auto sm = PruningHandler{bftEngine::ReplicaConfig::instance().pathToOperatorPublicKey_, - bftEngine::ReplicaConfig::instance().operatorMsgSigningAlgo, + auto sm = PruningHandler{ReplicaConfig::instance().pathToOperatorPublicKey_, + ReplicaConfig::instance().operatorMsgSigningAlgo, storage, storage, blocks_deleter, @@ -482,15 +514,14 @@ TEST_F(test_rocksdb, sm_latest_prunable_request_pruning_disabled) { replicaConfig.numBlocksToKeep_ = num_blocks_to_keep; replicaConfig.replicaId = replica_idx; replicaConfig.pruningEnabled_ = false; - replicaConfig.replicaPrivateKey = keyPair[1].first; + replicaConfig.replicaPrivateKey = s_eddsaKeyPairs[1].first; auto storage = TestStorage(db); auto &blocks_deleter = storage; - const auto verifier = PruningVerifier{replicaConfig.publicKeysOfReplicas}; InitBlockchainStorage(replica_count, storage); - auto sm = PruningHandler{bftEngine::ReplicaConfig::instance().pathToOperatorPublicKey_, - bftEngine::ReplicaConfig::instance().operatorMsgSigningAlgo, + auto sm = PruningHandler{ReplicaConfig::instance().pathToOperatorPublicKey_, + ReplicaConfig::instance().operatorMsgSigningAlgo, storage, storage, blocks_deleter, @@ -511,18 +542,18 @@ TEST_F(test_rocksdb, sm_handle_prune_request_on_pruning_disabled) { replicaConfig.numBlocksToKeep_ = num_blocks_to_keep; replicaConfig.replicaId = replica_idx; replicaConfig.pruningEnabled_ = false; - replicaConfig.replicaPrivateKey = keyPair[1].first; + replicaConfig.replicaPrivateKey = s_eddsaKeyPairs[1].first; TestStorage storage(db); auto &blocks_deleter = storage; - auto sm = PruningHandler{bftEngine::ReplicaConfig::instance().pathToOperatorPublicKey_, - bftEngine::ReplicaConfig::instance().operatorMsgSigningAlgo, + auto sm = PruningHandler{ReplicaConfig::instance().pathToOperatorPublicKey_, + ReplicaConfig::instance().operatorMsgSigningAlgo, storage, storage, blocks_deleter, false}; - const auto req = ConstructPruneRequest(client_idx, private_keys_of_replicas); + const auto req = ConstructPruneRequest(client_idx); concord::messages::ReconfigurationResponse rres; auto res = sm.handle(req, 0, UINT32_MAX, {}, rres); ASSERT_TRUE(res); @@ -535,20 +566,20 @@ TEST_F(test_rocksdb, sm_handle_correct_prune_request) { replicaConfig.numBlocksToKeep_ = num_blocks_to_keep; replicaConfig.replicaId = replica_idx; replicaConfig.pruningEnabled_ = true; - replicaConfig.replicaPrivateKey = keyPair[1].first; + replicaConfig.replicaPrivateKey = s_eddsaKeyPairs[1].first; TestStorage storage(db); auto &blocks_deleter = storage; InitBlockchainStorage(replica_count, storage); - auto sm = PruningHandler{bftEngine::ReplicaConfig::instance().pathToOperatorPublicKey_, - bftEngine::ReplicaConfig::instance().operatorMsgSigningAlgo, + auto sm = PruningHandler{ReplicaConfig::instance().pathToOperatorPublicKey_, + ReplicaConfig::instance().operatorMsgSigningAlgo, storage, storage, blocks_deleter, false}; const auto latest_prunable_block_id = storage.getLastBlockId() - num_blocks_to_keep; - const auto req = ConstructPruneRequest(client_idx, private_keys_of_replicas, latest_prunable_block_id); + const auto req = ConstructPruneRequest(client_idx, latest_prunable_block_id); blocks_deleter.deleteBlocksUntil(latest_prunable_block_id + 1); concord::messages::ReconfigurationResponse rres; auto res = sm.handle(req, 0, UINT32_MAX, {}, rres); @@ -564,13 +595,13 @@ TEST_F(test_rocksdb, sm_handle_incorrect_prune_request) { replicaConfig.numBlocksToKeep_ = num_blocks_to_keep; replicaConfig.replicaId = replica_idx; replicaConfig.pruningEnabled_ = true; - replicaConfig.replicaPrivateKey = keyPair[1].first; + replicaConfig.replicaPrivateKey = s_eddsaKeyPairs[1].first; TestStorage storage(db); auto &blocks_deleter = storage; InitBlockchainStorage(replica_count, storage); - auto sm = PruningHandler{bftEngine::ReplicaConfig::instance().pathToOperatorPublicKey_, - bftEngine::ReplicaConfig::instance().operatorMsgSigningAlgo, + auto sm = PruningHandler{ReplicaConfig::instance().pathToOperatorPublicKey_, + ReplicaConfig::instance().operatorMsgSigningAlgo, storage, storage, blocks_deleter, @@ -578,7 +609,7 @@ TEST_F(test_rocksdb, sm_handle_incorrect_prune_request) { // Add a valid N + 1 latest prunable block. { - auto req = ConstructPruneRequest(client_idx, private_keys_of_replicas); + auto req = ConstructPruneRequest(client_idx); const auto &block = req.latest_prunable_block[3]; auto latest_prunnable_block = concord::messages::LatestPrunableBlock(); latest_prunnable_block.block_id = block.block_id; @@ -594,7 +625,7 @@ TEST_F(test_rocksdb, sm_handle_incorrect_prune_request) { // Send N - 1 latest prunable blocks. { - auto req = ConstructPruneRequest(client_idx, private_keys_of_replicas); + auto req = ConstructPruneRequest(client_idx); req.latest_prunable_block.pop_back(); concord::messages::ReconfigurationResponse rres; auto res = sm.handle(req, 0, UINT32_MAX, {}, rres); @@ -605,7 +636,7 @@ TEST_F(test_rocksdb, sm_handle_incorrect_prune_request) { // Send a latest prunable block with an invalid signature. { - auto req = ConstructPruneRequest(client_idx, private_keys_of_replicas); + auto req = ConstructPruneRequest(client_idx); auto &block = req.latest_prunable_block[req.latest_prunable_block.size() - 1]; block.signature[0] += 1; concord::messages::ReconfigurationResponse rres; @@ -618,7 +649,6 @@ TEST_F(test_rocksdb, sm_handle_incorrect_prune_request) { } // namespace int main(int argc, char **argv) { - setUpKeysConfiguration_4(); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/libs/log/logger.hpp b/libs/log/logger.hpp index 93294cc991..5d6d417743 100644 --- a/libs/log/logger.hpp +++ b/libs/log/logger.hpp @@ -35,6 +35,7 @@ extern logging::Logger THRESHSIGN_LOG; extern logging::Logger OPENSSL_LOG; extern logging::Logger BLS_LOG; extern logging::Logger EDDSA_MULTISIG_LOG; +extern logging::Logger PRUNING_LOG; extern logging::Logger EDDSA_SIG_LOG; extern logging::Logger KEY_EX_LOG; extern logging::Logger CAT_BLOCK_LOG; diff --git a/libs/log/src/logger.cpp b/libs/log/src/logger.cpp index c4703c8321..c35ab38008 100644 --- a/libs/log/src/logger.cpp +++ b/libs/log/src/logger.cpp @@ -30,6 +30,7 @@ logging::Logger CNSUS = logging::getLogger("concord.bft.consensus"); logging::Logger THRESHSIGN_LOG = logging::getLogger("concord.bft.threshsign"); logging::Logger BLS_LOG = logging::getLogger("concord.bft.threshsign.bls"); logging::Logger EDDSA_MULTISIG_LOG = logging::getLogger("threshsign.eddsa"); +logging::Logger PRUNING_LOG = logging::getLogger("concord.pruning"); logging::Logger OPENSSL_LOG = logging::getLogger("concord.bft.openssl"); logging::Logger EDDSA_SIG_LOG = logging::getLogger("singlesig.eddsa"); logging::Logger KEY_EX_LOG = logging::getLogger("concord.bft.key-exchange"); diff --git a/tests/apollo/test_skvbc_commit_path.py b/tests/apollo/test_skvbc_commit_path.py index 233dd969bd..cd8a74cb13 100644 --- a/tests/apollo/test_skvbc_commit_path.py +++ b/tests/apollo/test_skvbc_commit_path.py @@ -271,4 +271,4 @@ async def test_fast_path_after_view_change(self, bft_network, tracker): await bft_network.wait_for_consensus_path( path_type=ConsensusPathType.OPTIMISTIC_FAST, run_ops=lambda: skvbc.send_n_kvs_sequentially(int(1.1 * EVALUATION_PERIOD_SEQUENCES)), - threshold=num_ops) \ No newline at end of file + threshold=num_ops) diff --git a/tests/apollo/test_skvbc_view_change.py b/tests/apollo/test_skvbc_view_change.py index 486bd8997e..33a2187173 100644 --- a/tests/apollo/test_skvbc_view_change.py +++ b/tests/apollo/test_skvbc_view_change.py @@ -480,8 +480,6 @@ async def test_multiple_vc_slow_path(self, bft_network, tracker): await bft_network.wait_for_view_with_threshold(view) await skvbc.read_your_writes() - await bft_network.assert_slow_path_prevalent(0, 0, random.choice( - bft_network.all_replicas(without=crashed_replicas))) @with_trio @with_bft_network(start_replica_cmd, diff --git a/tests/apollo/util/bft.py b/tests/apollo/util/bft.py index b73f6529fe..bea7ee5a11 100644 --- a/tests/apollo/util/bft.py +++ b/tests/apollo/util/bft.py @@ -247,7 +247,7 @@ async def test_with_bft_network(): return decorator MAX_MSG_SIZE = 64*1024 # 64k -REQ_TIMEOUT_MILLI = 5000 * 2 +REQ_TIMEOUT_MILLI = 5000 RETRY_TIMEOUT_MILLI = 250 METRICS_TIMEOUT_SEC = 5 @@ -416,7 +416,7 @@ def new(cls, config, background_nursery, client_factory=None, with_cre=False, us return bft_network @classmethod - def existing(cls, config, replicas, clients, client_factory=None, background_nursery=None, builddir=None): + def existing(cls, config, replicas, clients, client_factory=None, background_nursery=None): certdir = None builddir = tempfile.mkdtemp(prefix='builddir') if not client_factory: diff --git a/tests/apollo/util/eliot_logging.py b/tests/apollo/util/eliot_logging.py index 91598e004b..cc9ed8d5ec 100644 --- a/tests/apollo/util/eliot_logging.py +++ b/tests/apollo/util/eliot_logging.py @@ -34,9 +34,6 @@ def set_file_destination(): now = logdir_timestamp() test_name = f"apollo_run_{now}" - if os.environ.get('BLOCKCHAIN_VERSION', default="1").lower() == "4": - test_name = test_name + "_v4" - relative_apollo_logs = 'tests/apollo/logs' relative_current_run_logs = f'{relative_apollo_logs}/{logdir_timestamp()}' logs_dir = f'../../build/{relative_current_run_logs}' @@ -56,8 +53,6 @@ def set_file_destination(): latest_shortcut.unlink() latest_shortcut.symlink_to(target=Path(f'../{relative_current_run_logs}'), target_is_directory=True) - # Set the log file path - to_file(open(test_log, "a+")) test_log_file = open(test_log, "a+") to_file(test_log_file) diff --git a/tests/apollo/util/pyclient/bft_client.py b/tests/apollo/util/pyclient/bft_client.py index 31ccb2fc1b..9a34e666ed 100644 --- a/tests/apollo/util/pyclient/bft_client.py +++ b/tests/apollo/util/pyclient/bft_client.py @@ -299,7 +299,7 @@ async def write_batch(self, msg_batch, batch_seq_nums=None, m_of_n_quorum=None, self.client_id, msg_seq_num, False, self.config.req_timeout_milli, batch_cid, msg, 0, True, reconfiguration=False, span_context=b'', signature=signature, batch_index=n)]) - data = bft_msgs.pack_batch_request(self.client_id, batch_size, msg_data, batch_batch_cid) + data = bft_msgs.pack_batch_request(self.client_id, batch_size, msg_data, batch_cid) if m_of_n_quorum is None: m_of_n_quorum = MofNQuorum.LinearizableQuorum(self.config, [r.id for r in self.replicas]) @@ -311,7 +311,7 @@ async def write_batch(self, msg_batch, batch_seq_nums=None, m_of_n_quorum=None, return await self._send_receive_loop(data, False, m_of_n_quorum, batch_size * self.config.retry_timeout_milli / 1000, no_retries=no_retries) except trio.TooSlowError: - raise trio.TooSlowError(f"client_id {self.client_id}, for batch msg {batch_batch_cid} {batch_seq_nums}") + raise trio.TooSlowError(f"client_id {self.client_id}, for batch msg {batch_cid} {batch_seq_nums}") finally: pass diff --git a/tests/simpleTest/src/simple_test_replica.cpp b/tests/simpleTest/src/simple_test_replica.cpp index ca1dba63e8..4d4af80abb 100644 --- a/tests/simpleTest/src/simple_test_replica.cpp +++ b/tests/simpleTest/src/simple_test_replica.cpp @@ -25,6 +25,17 @@ SimpleTestReplica::SimpleTestReplica(ICommunication *commObject, MetadataStorage *metaDataStorage) : comm{commObject}, replicaConfig{rc}, behaviorPtr{behvPtr}, statePtr(state) { bftEngine::IControlHandler::instance(new bftEngine::ControlHandler()); + + ReplicasInfo repsInfo{rc, dynamicCollectorForPartialProofs, dynamicCollectorForExecutionProofs}; + SigManager::init(rc.replicaId, + rc.replicaPrivateKey, + rc.publicKeysOfReplicas, + concord::crypto::KeyFormat::HexaDecimalStrippedFormat, + rc.getPublicKeysOfClients(), + concord::crypto::KeyFormat::PemFormat, + {{repsInfo.getIdOfOperator(), rc.getOperatorPublicKey(), concord::crypto::KeyFormat::PemFormat}}, + repsInfo); + replica = bftEngine::ReplicaFactory::createReplica(rc, std::shared_ptr(state), inMemoryST,