diff --git a/.gitignore b/.gitignore index f0f787aaf8..6ebbe836f6 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ coverage.txt tools-stamp __debug_bin profile.out +testing/e2e/networks/*/ diff --git a/go.mod b/go.mod index 3b51716d4a..75bd517317 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,19 @@ require ( gopkg.in/yaml.v2 v2.4.0 ) +require ( + github.com/Microsoft/go-winio v0.6.0 // indirect + github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0-rc2 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect + golang.org/x/mod v0.9.0 // indirect + golang.org/x/tools v0.7.0 // indirect + gotest.tools/v3 v3.4.0 // indirect +) + require ( cloud.google.com/go v0.105.0 // indirect cloud.google.com/go/compute v1.12.1 // indirect @@ -47,6 +60,7 @@ require ( filippo.io/edwards25519 v1.0.0-rc.1 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.1 // indirect + github.com/BurntSushi/toml v1.2.1 github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d // indirect github.com/StackExchange/wmi v1.2.1 // indirect github.com/Workiva/go-datastructures v1.0.53 // indirect @@ -80,6 +94,7 @@ require ( github.com/dgraph-io/badger/v2 v2.2007.4 // indirect github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect + github.com/docker/docker v20.10.21+incompatible github.com/dustin/go-humanize v1.0.1-0.20200219035652-afde56e7acac // indirect github.com/dvsekhvalnov/jose2go v1.5.0 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect diff --git a/go.sum b/go.sum index f149a2ed47..55175029ab 100644 --- a/go.sum +++ b/go.sum @@ -68,6 +68,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSu github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d h1:nalkkPQcITbvhmL4+C4cKA87NW0tfm3Kl9VXRoPywFg= github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d/go.mod h1:URdX5+vg25ts3aCh8H5IFZybJYKWhJHYMTnf+ULtoC4= @@ -77,6 +79,7 @@ github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwS github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= +github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -294,9 +297,15 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.21+incompatible h1:UTLdBmHk3bEY+w8qeO5KttOhy6OmXWsl/FEet9Uswog= +github.com/docker/docker v20.10.21+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dop251/goja v0.0.0-20211011172007-d99e4b8cbf48/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -740,6 +749,7 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= +github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -747,6 +757,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= @@ -787,7 +798,9 @@ github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= +github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= github.com/opencontainers/runc v1.1.3 h1:vIXrkId+0/J2Ymu2m7VjGvbSlAId9XNRPhn2p4b+d8w= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= @@ -887,8 +900,8 @@ github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.27.0 h1:1T7qCieN22GVc8S4Q2yuexzBb1EqjbgjSH9RohbMjKs= github.com/rs/zerolog v1.27.0/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U= -github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -910,6 +923,7 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -1117,6 +1131,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1276,6 +1291,7 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= @@ -1363,6 +1379,7 @@ golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1530,6 +1547,8 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= +gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/testing/e2e/Makefile b/testing/e2e/Makefile new file mode 100644 index 0000000000..1b63d30a3b --- /dev/null +++ b/testing/e2e/Makefile @@ -0,0 +1,9 @@ +all: cli docker + +cli: + go build -o build/e2e ./cmd/e2e + +docker: + docker build --tag ghcr.io/celestiaorg/celestia-app:current -f ../../Dockerfile ../.. + +PHONY: all cli docker diff --git a/testing/e2e/README.md b/testing/e2e/README.md new file mode 100644 index 0000000000..56b6c75232 --- /dev/null +++ b/testing/e2e/README.md @@ -0,0 +1,43 @@ +# E2E Framework + +## Purpose + +The e2e package provides a framework for integration testing of the Celestia consensus network. +It consists of a simple CLI and a series of TOML testnet files which manage the running of +several instances within the same network. The e2e test suite has the following purposes in mind: + +- **Compatibility testing**: Ensuring that multiple minor versions can operate successfully together within the same network. +- **Upgrade testing**: Ensure upgrades, whether major or minor, can perform seamlessly. +- **Sync testing**: Ensure that the latest version can sync data from the entire chain. +- **Non determinism check**: Ensure that the state machine is free of non-determinism that could compromise replication. +- **Invariant checking**: Ensure that system wide invariants hold. + +The e2e package is designed predominantly for correctness based testing of small clusters of node. +It is designed to be relatively quick and can be used locally. It relies on docker and docker compose +to orchestrate the nodes. + +## Usage + +To get started, run `make` within the e2e package directory. This builds the image referring to the current +branch as well as the cli (To build just the cli run `make cli`). Then, to run the complete suite: + +```bash +./build/e2e -f networks/simple.toml +``` + +You should see something like + +```bash +Setting up network simple-56602 +Spinning up testnet +Starting validator01 on +Starting validator02 on +Starting validator03 on +Starting validator04 on +Starting full01 on +Waiting for the network to reach height 20 +Stopping testnet +Finished testnet successfully +``` + +Alternatively you can use the commands: `setup`, `start`, `stop`, and `cleanup`. diff --git a/testing/e2e/cmd/e2e/main.go b/testing/e2e/cmd/e2e/main.go new file mode 100644 index 0000000000..f8b8f53f3b --- /dev/null +++ b/testing/e2e/cmd/e2e/main.go @@ -0,0 +1,124 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + + e2e "github.com/celestiaorg/celestia-app/testing/e2e/pkg" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +func main() { + NewCLI().Run() +} + +type CLI struct { + root *cobra.Command + testnet *e2e.Testnet +} + +func NewCLI() *CLI { + cli := &CLI{} + cli.root = &cobra.Command{ + Use: "e2e", + Short: "Command line runner for celestia app e2e framework", + SilenceUsage: true, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + file, err := cmd.Flags().GetString("file") + if err != nil { + return err + } + + manifest, err := e2e.LoadManifest(file) + if err != nil { + return fmt.Errorf("loading manifest: %w", err) + } + + testnet, err := e2e.LoadTestnet(manifest, file) + if err != nil { + return fmt.Errorf("building testnet: %w", err) + } + + cli.testnet = testnet + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + if err := e2e.Cleanup(ctx, cli.testnet); err != nil { + return fmt.Errorf("preparing testnet: %w", err) + } + + if err := e2e.Setup(ctx, cli.testnet); err != nil { + return fmt.Errorf("setting up testnet: %w", err) + } + defer func() { _ = e2e.Cleanup(ctx, cli.testnet) }() + + if err := e2e.Start(ctx, cli.testnet); err != nil { + return fmt.Errorf("starting network: %w", err) + } + + if err := e2e.WaitForNBlocks(ctx, cli.testnet, 10); err != nil { + return fmt.Errorf("waiting for the network to produce blocks: %w", err) + } + + if err := e2e.Stop(ctx, cli.testnet); err != nil { + return fmt.Errorf("stopping network: %w", err) + } + + fmt.Println("Finished testnet successfully") + return nil + }, + } + cli.root.PersistentFlags().StringP("file", "f", "", "Testnet TOML manifest") + _ = cli.root.MarkPersistentFlagRequired("file") + + cli.root.AddCommand(&cobra.Command{ + Use: "setup", + Short: "Setups a testnet", + RunE: func(cmd *cobra.Command, args []string) error { + return e2e.Setup(cmd.Context(), cli.testnet) + }, + }) + + cli.root.AddCommand(&cobra.Command{ + Use: "start", + Short: "Starts a testnet", + RunE: func(cmd *cobra.Command, args []string) error { + return e2e.Start(cmd.Context(), cli.testnet) + }, + }) + + cli.root.AddCommand(&cobra.Command{ + Use: "stop", + Short: "Stops a testnet", + RunE: func(cmd *cobra.Command, args []string) error { + return e2e.Stop(cmd.Context(), cli.testnet) + }, + }) + + cli.root.AddCommand(&cobra.Command{ + Use: "cleanup", + Short: "Tears down network and removes all resources", + RunE: func(cmd *cobra.Command, args []string) error { + return e2e.Cleanup(cmd.Context(), cli.testnet) + }, + }) + + return cli +} + +func (cli *CLI) Run() { + ctx, cancel := signal.NotifyContext(context.Background(), + syscall.SIGINT, + syscall.SIGTERM, + ) + defer cancel() + if err := cli.root.ExecuteContext(ctx); err != nil { + log.Err(err) + os.Exit(1) + } +} diff --git a/testing/e2e/networks/simple.toml b/testing/e2e/networks/simple.toml new file mode 100644 index 0000000000..26ef084c64 --- /dev/null +++ b/testing/e2e/networks/simple.toml @@ -0,0 +1,13 @@ +[node.validator01] +self_delegation = 1000000 +[node.validator02] +self_delegation = 1000000 +[node.validator03] +self_delegation = 1000000 +[node.validator04] +self_delegation = 1000000 +[node.full01] +start_height = 10 + +[account.user1] +[account.user2] \ No newline at end of file diff --git a/testing/e2e/pkg/exec.go b/testing/e2e/pkg/exec.go new file mode 100644 index 0000000000..2410b53193 --- /dev/null +++ b/testing/e2e/pkg/exec.go @@ -0,0 +1,33 @@ +package e2e + +import ( + "fmt" + osexec "os/exec" + "path/filepath" +) + +// execute executes a shell command. +func exec(args ...string) error { + cmd := osexec.Command(args[0], args[1:]...) + out, err := cmd.CombinedOutput() + switch err := err.(type) { + case nil: + return nil + case *osexec.ExitError: + return fmt.Errorf("failed to run %q:\n%v", args, string(out)) + default: + return err + } +} + +// execCompose runs a Docker Compose command for a testnet. +func execCompose(dir string, args ...string) error { + return exec(append( + []string{"docker compose", "-f", filepath.Join(dir, "docker-compose.yml")}, + args...)...) +} + +// execDocker runs a Docker command. +func execDocker(args ...string) error { + return exec(append([]string{"docker"}, args...)...) +} diff --git a/testing/e2e/pkg/lifecycle.go b/testing/e2e/pkg/lifecycle.go new file mode 100644 index 0000000000..0f169b498f --- /dev/null +++ b/testing/e2e/pkg/lifecycle.go @@ -0,0 +1,106 @@ +package e2e + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" +) + +// Start commences the testnet. +func Start(ctx context.Context, testnet *Testnet) error { + fmt.Println("Spinning up testnet") + nodes := testnet.NodesByStartHeight() + for _, node := range nodes { + + if node.StartHeight != 0 { + if err := WaitForHeight(ctx, testnet, node.StartHeight); err != nil { + return err + } + } + + fmt.Printf("Starting %s at height %d on %s\n", node.Name, node.StartHeight, fmt.Sprintf("http://localhost:%d", node.ProxyPort)) + + if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil { + return err + } + } + + return nil +} + +// Stop stops the currently running network +func Stop(_ context.Context, testnet *Testnet) error { + fmt.Println("Stopping testnet") + return execCompose(testnet.Dir, "down") +} + +// Cleanup removes the Docker Compose containers and testnet directory. +func Cleanup(_ context.Context, testnet *Testnet) error { + err := cleanupDocker() + if err != nil { + return err + } + err = cleanupDir(testnet.Dir) + if err != nil { + return err + } + return nil +} + +// cleanupDocker removes all E2E resources (with label e2e=True), regardless +// of testnet. +func cleanupDocker() error { + // GNU xargs requires the -r flag to not run when input is empty, macOS + // does this by default. Ugly, but works. + xargsR := `$(if [[ $OSTYPE == "linux-gnu"* ]]; then echo -n "-r"; fi)` + + err := exec("bash", "-c", fmt.Sprintf( + "docker container ls -qa --filter label=e2e | xargs %v docker container rm -f", xargsR)) + if err != nil { + return err + } + + err = exec("bash", "-c", fmt.Sprintf( + "docker network ls -q --filter label=e2e | xargs %v docker network rm", xargsR)) + if err != nil { + return err + } + + return nil +} + +// cleanupDir cleans up a testnet directory +func cleanupDir(dir string) error { + if dir == "" { + return errors.New("no directory set") + } + + _, err := os.Stat(dir) + if os.IsNotExist(err) { + return nil + } else if err != nil { + return err + } + + // On Linux, some local files in the volume will be owned by root since Tendermint + // runs as root inside the container, so we need to clean them up from within a + // container running as root too. + absDir, err := filepath.Abs(dir) + if err != nil { + return err + } + err = execDocker("run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir), + "tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/") + if err != nil { + return err + } + + err = os.RemoveAll(dir) + if err != nil { + return err + } + + return nil +} diff --git a/testing/e2e/pkg/manifest.go b/testing/e2e/pkg/manifest.go new file mode 100644 index 0000000000..82d64f2f89 --- /dev/null +++ b/testing/e2e/pkg/manifest.go @@ -0,0 +1,70 @@ +package e2e + +import ( + "fmt" + "os" + + "github.com/BurntSushi/toml" +) + +type Manifest struct { + Nodes map[string]*ManifestNode `toml:"node"` + Accounts map[string]*ManifestAccount `toml:"account"` +} + +type ManifestNode struct { + // Versions is an array of binary versions that the node + // will run in it's lifetime. A series of upgrades can be + // triggered by the upgrade command or will happen automatically + // across a testnet. + // + // Version strings are used to pull docker images from ghcr.io. + // Alternatively, use "current", if you want to use the binary + // based on the current branch. You must have the image built by + // running "make docker". Default set to "current" + Versions []string `toml:"versions"` + + // The height that the node will start at + StartHeight int64 `toml:"start_height"` + + // Peers are the set of peers that initially populate the address + // book. Persistent peers and seeds are currently not supported + // By default all nodes declared in the manifest are included + Peers []string `toml:"peers"` + + // SelfDelegation is the delegation of the validator when they + // first come up. If set to 0, the node is not considered a validator + SelfDelegation int64 `toml:"self_delegation"` +} + +// ManifestAccounts represent SDK accounts that sign and +// submit transactions. If the account has the same name as +// the node it is the operator address for the validator. +// Unless specified it will have a default self delegation +// All accounts specfied are created at genesis. +type ManifestAccount struct { + // Tokens symbolizes the genesis supply of liquid tokens to that account + Tokens int64 `toml:"tokens"` + + // The key type to derive the account key from. Defaults to secp256k1 + KeyType string `toml:"key_type"` +} + +// Save saves the testnet manifest to a file. +func (m Manifest) Save(file string) error { + f, err := os.Create(file) + if err != nil { + return fmt.Errorf("failed to create manifest file %q: %w", file, err) + } + return toml.NewEncoder(f).Encode(m) +} + +// LoadManifest loads a testnet manifest from a file. +func LoadManifest(file string) (Manifest, error) { + manifest := Manifest{} + _, err := toml.DecodeFile(file, &manifest) + if err != nil { + return manifest, fmt.Errorf("failed to load testnet manifest %q: %w", file, err) + } + return manifest, nil +} diff --git a/testing/e2e/pkg/rpc.go b/testing/e2e/pkg/rpc.go new file mode 100644 index 0000000000..b0fe9daf01 --- /dev/null +++ b/testing/e2e/pkg/rpc.go @@ -0,0 +1,130 @@ +package e2e + +import ( + "context" + "errors" + "fmt" + "sort" + "time" + + rpchttp "github.com/tendermint/tendermint/rpc/client/http" +) + +const ( + waitForHeightTimeout = 20 * time.Second + maxTimePerBlock = 20 * time.Second +) + +// WaitForNBlocks queries the current latest height and waits until the network +// progresses to "blocks" blocks ahead. +func WaitForNBlocks(ctx context.Context, testnet *Testnet, blocks int64) error { + height, err := GetHeights(ctx, testnet) + if err != nil { + return err + } + + fmt.Printf("Waiting for the network to reach height %d\n", height[0]+blocks) + + return WaitForHeight(ctx, testnet, height[0]+blocks) +} + +// WaitForHeight waits until the first node reaches the height specified. If +// no progress is made within a 20 second window then the function times out +// with an error. +func WaitForHeight(ctx context.Context, testnet *Testnet, height int64) error { + var ( + err error + maxHeight int64 + clients = map[string]*rpchttp.HTTP{} + lastIncrease = time.Now() + ) + + for { + for _, node := range testnet.Nodes { + if node.StartHeight > height { + continue + } + client, ok := clients[node.Name] + if !ok { + client, err = node.Client() + if err != nil { + continue + } + clients[node.Name] = client + } + subctx, cancel := context.WithTimeout(ctx, 1*time.Second) + defer cancel() + + // request the status of the node + result, err := client.Status(subctx) + if err != nil { + if errors.Is(err, context.DeadlineExceeded) { + return err + } + continue + } + + if result.SyncInfo.LatestBlockHeight >= height { + return nil + } + + if result.SyncInfo.LatestBlockHeight > maxHeight { + maxHeight = result.SyncInfo.LatestBlockHeight + lastIncrease = time.Now() + } + + // If no progress has been made in the last 20 seconds, return an error. + if time.Since(lastIncrease) > waitForHeightTimeout { + return fmt.Errorf("network unable to reach next height within %s (max height: %d, target height: %d)", + waitForHeightTimeout, maxHeight, height, + ) + } + + time.Sleep(1 * time.Second) + } + } +} + +// GetHeights loops through all running nodes and returns an array of heights +// in order of highest to lowest. +func GetHeights(ctx context.Context, testnet *Testnet) ([]int64, error) { + var ( + err error + heights = make([]int64, 0, len(testnet.Nodes)) + clients = map[string]*rpchttp.HTTP{} + ) + + for _, node := range testnet.Nodes { + client, ok := clients[node.Name] + if !ok { + client, err = node.Client() + if err != nil { + continue + } + clients[node.Name] = client + } + subctx, cancel := context.WithTimeout(ctx, 1*time.Second) + defer cancel() + + // request the status of the node + result, err := client.Status(subctx) + if err != nil { + if errors.Is(err, context.DeadlineExceeded) { + return nil, err + } + continue + } + + heights = append(heights, result.SyncInfo.LatestBlockHeight) + } + if len(heights) == 0 { + return nil, errors.New("network is not running") + } + + // return heights in descending order + sort.Slice(heights, func(i, j int) bool { + return heights[i] > heights[j] + }) + + return heights, nil +} diff --git a/testing/e2e/pkg/setup.go b/testing/e2e/pkg/setup.go new file mode 100644 index 0000000000..fbd601793f --- /dev/null +++ b/testing/e2e/pkg/setup.go @@ -0,0 +1,351 @@ +package e2e + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "html/template" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/celestiaorg/celestia-app/app" + "github.com/celestiaorg/celestia-app/app/encoding" + codectypes "github.com/cosmos/cosmos-sdk/codec/types" + cryptocodec "github.com/cosmos/cosmos-sdk/crypto/codec" + serverconfig "github.com/cosmos/cosmos-sdk/server/config" + sdk "github.com/cosmos/cosmos-sdk/types" + auth "github.com/cosmos/cosmos-sdk/x/auth/types" + bank "github.com/cosmos/cosmos-sdk/x/bank/types" + slashing "github.com/cosmos/cosmos-sdk/x/slashing/types" + staking "github.com/cosmos/cosmos-sdk/x/staking/types" + dockertypes "github.com/docker/docker/api/types" + "github.com/docker/docker/client" + "github.com/ethereum/go-ethereum/common" + "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/p2p" + "github.com/tendermint/tendermint/p2p/pex" + "github.com/tendermint/tendermint/privval" + "github.com/tendermint/tendermint/types" +) + +func Setup(ctx context.Context, testnet *Testnet) error { + // Ensure that all the requisite images are available + if err := SetupImages(ctx, testnet); err != nil { + return fmt.Errorf("setting up images: %w", err) + } + + fmt.Printf("Setting up network %s\n", testnet.Name) + + _, err := os.Stat(testnet.Dir) + if err == nil { + return errors.New("testnet directory already exists") + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("checking if testnet directory exists: %w", err) + } + + // Create the directory for the testnet + if err := os.MkdirAll(testnet.Dir, os.ModePerm); err != nil { + return err + } + cleanup := func() { os.RemoveAll(testnet.Dir) } + + // Create the docker compose file + if err := WriteDockerCompose(testnet, filepath.Join(testnet.Dir, "docker-compose.yml")); err != nil { + cleanup() + return fmt.Errorf("setting up docker compose: %w", err) + } + + // Make the genesis file for the testnet + genesis, err := MakeGenesis(testnet) + if err != nil { + cleanup() + return fmt.Errorf("making genesis: %w", err) + } + + // Initialize the file system and configs for each node + for name, node := range testnet.Nodes { + err := InitNode(node, genesis, testnet.Dir) + if err != nil { + cleanup() + return fmt.Errorf("initializing node %s: %w", name, err) + } + } + + return nil +} + +// SetupImages ensures that all the requisite docker images for each +// used celestia consensus version. +func SetupImages(ctx context.Context, testnet *Testnet) error { + c, err := client.NewClientWithOpts() + if err != nil { + return fmt.Errorf("establishing docker client: %v", err) + } + + versions := testnet.GetAllVersions() + + for _, v := range versions { + if v == "current" { + // we assume that the user has locally downloaded + // the current docker image + continue + } + refStr := dockerSrcURL + ":" + v + fmt.Printf("Pulling in docker image: %s\n", refStr) + rc, err := c.ImagePull(ctx, refStr, dockertypes.ImagePullOptions{}) + if err != nil { + return fmt.Errorf("error pulling image %s: %w", refStr, err) + } + _, _ = io.Copy(io.Discard, rc) + _ = rc.Close() + } + + return nil +} + +func MakeGenesis(testnet *Testnet) (types.GenesisDoc, error) { + encCdc := encoding.MakeConfig(app.ModuleEncodingRegisters...) + appGenState := app.ModuleBasics.DefaultGenesis(encCdc.Codec) + bankGenesis := bank.DefaultGenesisState() + stakingGenesis := staking.DefaultGenesisState() + slashingGenesis := slashing.DefaultGenesisState() + genAccs := []auth.GenesisAccount{} + stakingGenesis.Params.BondDenom = app.BondDenom + delegations := make([]staking.Delegation, 0, len(testnet.Nodes)) + valInfo := make([]slashing.SigningInfo, 0, len(testnet.Nodes)) + balances := make([]bank.Balance, 0, len(testnet.Accounts)+1) + var ( + validators staking.Validators + totalBonded int64 + ) + + // setup the validator information on the state machine + for name, node := range testnet.Nodes { + if !node.IsValidator() || node.StartHeight != 0 { + continue + } + + addr := node.AccountKey.PubKey().Address() + pk, err := cryptocodec.FromTmPubKeyInterface(node.SignerKey.PubKey()) + if err != nil { + return types.GenesisDoc{}, fmt.Errorf("converting public key for node %s: %w", node.Name, err) + } + pkAny, err := codectypes.NewAnyWithValue(pk) + if err != nil { + return types.GenesisDoc{}, err + } + evmAddress := common.HexToAddress(crypto.CRandHex(common.AddressLength)) + + validators = append(validators, staking.Validator{ + OperatorAddress: sdk.ValAddress(addr).String(), + ConsensusPubkey: pkAny, + Description: staking.Description{ + Moniker: name, + }, + Status: staking.Bonded, + Tokens: sdk.NewInt(node.SelfDelegation), + DelegatorShares: sdk.OneDec(), + // 5% commission + Commission: staking.NewCommission(sdk.NewDecWithPrec(5, 2), sdk.OneDec(), sdk.OneDec()), + MinSelfDelegation: sdk.ZeroInt(), + EvmAddress: evmAddress.Hex(), + }) + totalBonded += node.SelfDelegation + consensusAddr := pk.Address() + delegations = append(delegations, staking.NewDelegation(sdk.AccAddress(addr), sdk.ValAddress(addr), sdk.OneDec())) + valInfo = append(valInfo, slashing.SigningInfo{ + Address: sdk.ConsAddress(consensusAddr).String(), + ValidatorSigningInfo: slashing.NewValidatorSigningInfo(sdk.ConsAddress(consensusAddr), 1, 0, time.Unix(0, 0), false, 0), + }) + } + stakingGenesis.Delegations = delegations + stakingGenesis.Validators = validators + slashingGenesis.SigningInfos = valInfo + + accountNumber := uint64(0) + for _, account := range testnet.Accounts { + pk, err := cryptocodec.FromTmPubKeyInterface(account.Key.PubKey()) + if err != nil { + return types.GenesisDoc{}, fmt.Errorf("converting public key for account %s: %w", account.Name, err) + } + + addr := pk.Address() + acc := auth.NewBaseAccount(addr.Bytes(), pk, accountNumber, 0) + genAccs = append(genAccs, acc) + balances = append(balances, bank.Balance{ + Address: sdk.AccAddress(addr).String(), + Coins: sdk.NewCoins( + sdk.NewCoin(app.BondDenom, sdk.NewInt(account.Tokens)), + ), + }) + } + // add bonded amount to bonded pool module account + balances = append(balances, bank.Balance{ + Address: auth.NewModuleAddress(staking.BondedPoolName).String(), + Coins: sdk.Coins{sdk.NewCoin(app.BondDenom, sdk.NewInt(totalBonded))}, + }) + bankGenesis.Balances = bank.SanitizeGenesisBalances(balances) + authGenesis := auth.NewGenesisState(auth.DefaultParams(), genAccs) + + // update the original genesis state + appGenState[bank.ModuleName] = encCdc.Codec.MustMarshalJSON(bankGenesis) + appGenState[auth.ModuleName] = encCdc.Codec.MustMarshalJSON(authGenesis) + appGenState[staking.ModuleName] = encCdc.Codec.MustMarshalJSON(stakingGenesis) + appGenState[slashing.ModuleName] = encCdc.Codec.MustMarshalJSON(slashingGenesis) + + if err := app.ModuleBasics.ValidateGenesis(encCdc.Codec, encCdc.TxConfig, appGenState); err != nil { + return types.GenesisDoc{}, fmt.Errorf("validating genesis: %w", err) + } + + appState, err := json.MarshalIndent(appGenState, "", " ") + if err != nil { + return types.GenesisDoc{}, fmt.Errorf("marshaling app state: %w", err) + } + + // Validator set and app hash are set in InitChain + return types.GenesisDoc{ + ChainID: testnet.Name, + GenesisTime: time.Now().UTC(), + ConsensusParams: types.DefaultConsensusParams(), + AppState: appState, + // AppHash is not provided but computed after InitChain + }, nil +} + +func MakeConfig(node *Node) (*config.Config, error) { + cfg := config.DefaultConfig() + cfg.Moniker = node.Name + cfg.RPC.ListenAddress = "tcp://0.0.0.0:26657" + cfg.P2P.ExternalAddress = fmt.Sprintf("tcp://%v", node.AddressP2P(false)) + cfg.P2P.PersistentPeers = strings.Join(node.Peers, ",") + + // TODO: when we use adaptive timeouts, add a parameter in the testnet manifest + // to set block times + // FIXME: This values get overridden by the timeout consts in the app package. + // We should modify this if we want to quicken the time of the blocks. + cfg.Consensus.TimeoutPropose = 1000 * time.Millisecond + cfg.Consensus.TimeoutCommit = 300 * time.Millisecond + return cfg, nil +} + +func InitNode(node *Node, genesis types.GenesisDoc, rootDir string) error { + // Initialize file directories + nodeDir := filepath.Join(rootDir, node.Name) + for _, dir := range []string{ + filepath.Join(nodeDir, "config"), + filepath.Join(nodeDir, "data"), + } { + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return fmt.Errorf("error creating directory %s: %w", dir, err) + } + } + + // Create and write the config file + cfg, err := MakeConfig(node) + if err != nil { + return fmt.Errorf("making config: %w", err) + } + config.WriteConfigFile(filepath.Join(nodeDir, "config", "config.toml"), cfg) + + // Store the genesis file + err = genesis.SaveAs(filepath.Join(nodeDir, "config", "genesis.json")) + if err != nil { + return fmt.Errorf("saving genesis: %w", err) + } + + // Create the app.toml file + appConfig, err := MakeAppConfig(node) + if err != nil { + return fmt.Errorf("making app config: %w", err) + } + serverconfig.WriteConfigFile(filepath.Join(nodeDir, "config", "app.toml"), appConfig) + + // Store the node key for the p2p handshake + err = (&p2p.NodeKey{PrivKey: node.NetworkKey}).SaveAs(filepath.Join(nodeDir, "config", "node_key.json")) + if err != nil { + return err + } + + // Store the validator signer key for consensus + (privval.NewFilePV(node.SignerKey, + filepath.Join(nodeDir, "config", "priv_validator_key.json"), + filepath.Join(nodeDir, "data", "priv_validator_state.json"), + )).Save() + + return nil +} + +func WriteDockerCompose(testnet *Testnet, file string) error { + tmpl, err := template.New("docker-compose").Parse(`version: '2.4' + +networks: + {{ .Name }}: + labels: + e2e: true + driver: bridge + ipam: + driver: default + config: + - subnet: {{ .IP }} + +services: +{{- range .Nodes }} + {{ .Name }}: + labels: + e2e: true + container_name: {{ .Name }} + image: ghcr.io/celestiaorg/celestia-app:{{ index .Versions 0 }} + entrypoint: ["/bin/celestia-appd"] + command: ["start"] + init: true + ports: + - 26656 + - {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657 + - 6060 + - 9090 + - 1317 + volumes: + - ./{{ .Name }}:/home/celestia/.celestia-app + networks: + {{ $.Name }}: + ipv4_address: {{ .IP }} + +{{end}}`) + if err != nil { + return err + } + var buf bytes.Buffer + err = tmpl.Execute(&buf, testnet) + if err != nil { + return err + } + return os.WriteFile(file, buf.Bytes(), 0o644) +} + +func WriteAddressBook(peers []string, file string) error { + book := pex.NewAddrBook(file, true) + for _, peer := range peers { + addr, err := p2p.NewNetAddressString(peer) + if err != nil { + return fmt.Errorf("parsing peer address %s: %w", peer, err) + } + err = book.AddAddress(addr, addr) + if err != nil { + return fmt.Errorf("adding peer address %s: %w", peer, err) + } + } + book.Save() + return nil +} + +func MakeAppConfig(node *Node) (*serverconfig.Config, error) { + srvCfg := serverconfig.DefaultConfig() + srvCfg.MinGasPrices = fmt.Sprintf("0.001%s", app.BondDenom) + return srvCfg, srvCfg.ValidateBasic() +} diff --git a/testing/e2e/pkg/testnet.go b/testing/e2e/pkg/testnet.go new file mode 100644 index 0000000000..d7b15d7fda --- /dev/null +++ b/testing/e2e/pkg/testnet.go @@ -0,0 +1,316 @@ +package e2e + +import ( + "errors" + "fmt" + "io" + "math" + "math/rand" + "net" + "path/filepath" + "sort" + "strings" + + "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto/ed25519" + "github.com/tendermint/tendermint/crypto/secp256k1" + "github.com/tendermint/tendermint/rpc/client/http" +) + +const ( + secp256k1Type = "secp256k1" + ed25519Type = "ed25519" + networkIPv4 = "10.186.73.0/24" + firstProxyPort uint32 = 4201 + dockerSrcURL = "ghcr.io/celestiaorg/celestia-app" + randomSeed int64 = 589308084734268 + defaultAccountTokens = 1e6 + rpcPort = 26657 +) + +type Testnet struct { + Name string // also used as the chain-id + Dir string + IP *net.IPNet + Nodes map[string]*Node + Accounts map[string]*Account +} + +type Node struct { + Name string + Versions []string + StartHeight int64 + Peers []string + SignerKey crypto.PrivKey + NetworkKey crypto.PrivKey + AccountKey crypto.PrivKey + IP net.IP + ProxyPort uint32 + SelfDelegation int64 +} + +type Account struct { + Name string + Tokens int64 + Key crypto.PrivKey +} + +func LoadTestnet(manifest Manifest, file string) (*Testnet, error) { + // the directory that the toml file is located in + dir := strings.TrimSuffix(file, filepath.Ext(file)) + name := fmt.Sprintf("%s-%d", filepath.Base(dir), rand.Intn(math.MaxUint16)) + _, ipNet, err := net.ParseCIDR(networkIPv4) + if err != nil { + return nil, fmt.Errorf("invalid IP network address %q: %w", networkIPv4, err) + } + ipGen := newIPGenerator(ipNet) + keyGen := newKeyGenerator(randomSeed) + proxyPort := firstProxyPort + + testnet := &Testnet{ + Dir: dir, + Name: name, + Nodes: make(map[string]*Node), + Accounts: make(map[string]*Account), + IP: ipGen.Network(), + } + + // deterministically sort names in alphabetical order + nodeNames := []string{} + for name := range manifest.Nodes { + nodeNames = append(nodeNames, name) + } + sort.Strings(nodeNames) + + for _, name := range nodeNames { + nodeManifest := manifest.Nodes[name] + if _, ok := testnet.Nodes[name]; ok { + return nil, fmt.Errorf("duplicate node name %s", name) + } + node := &Node{ + Name: name, + Versions: nodeManifest.Versions, + StartHeight: nodeManifest.StartHeight, + Peers: nodeManifest.Peers, + SignerKey: keyGen.Generate(ed25519Type), + NetworkKey: keyGen.Generate(ed25519Type), + AccountKey: keyGen.Generate(secp256k1Type), + SelfDelegation: nodeManifest.SelfDelegation, + IP: ipGen.Next(), + ProxyPort: proxyPort, + } + if len(node.Versions) == 0 { + node.Versions = []string{"current"} + } + + testnet.Nodes[name] = node + proxyPort++ + } + + for name, node := range testnet.Nodes { + // fill up the peers field if it is empty + if len(node.Peers) == 0 { + for otherName := range testnet.Nodes { + if otherName == name { + continue + } + node.Peers = append(node.Peers, otherName) + } + } + // replace the peer names with the P2P address. + for idx, peer := range node.Peers { + node.Peers[idx] = testnet.Nodes[peer].AddressP2P(true) + } + } + + // deterministically sort accounts in alphabetical order + accountNames := []string{} + for name := range manifest.Accounts { + accountNames = append(accountNames, name) + } + sort.Strings(accountNames) + + for _, name := range accountNames { + accountManifest := manifest.Accounts[name] + if _, ok := testnet.Accounts[name]; ok { + return nil, fmt.Errorf("duplicate account name %s", name) + } + account := &Account{ + Name: name, + Tokens: accountManifest.Tokens, + Key: keyGen.Generate(accountManifest.KeyType), + } + if account.Tokens == 0 { + account.Tokens = defaultAccountTokens + } + testnet.Accounts[name] = account + } + + return testnet, testnet.Validate() +} + +func (t *Testnet) Validate() (err error) { + if len(t.Accounts) == 0 { + return errors.New("at least one account is required") + } + if len(t.Nodes) == 0 { + return errors.New("at least one node is required") + } + validators := 0 + for name, node := range t.Nodes { + if err := node.Validate(); err != nil { + return fmt.Errorf("invalid node %s: %w", name, err) + } + // must have at least one validator + if node.SelfDelegation > 0 { + validators++ + } + } + if validators == 0 { + return errors.New("at least one node must a validator by having an associated account") + } + for _, account := range t.Accounts { + if err := account.Validate(); err != nil { + return err + } + } + + return nil +} + +func (t *Testnet) GetAllVersions() []string { + versions := make(map[string]struct{}) + // filter duplicate version strings + for _, node := range t.Nodes { + for _, version := range node.Versions { + versions[version] = struct{}{} + } + } + + // convert to list + versionsList := []string{} + for version := range versions { + versionsList = append(versionsList, version) + } + return versionsList +} + +func (t *Testnet) NodesByStartHeight() []*Node { + nodes := make([]*Node, 0, len(t.Nodes)) + for _, node := range t.Nodes { + nodes = append(nodes, node) + } + sort.Slice(nodes, func(i, j int) bool { + if nodes[i].StartHeight == nodes[j].StartHeight { + return nodes[i].Name < nodes[j].Name + } + return nodes[i].StartHeight < nodes[j].StartHeight + }) + return nodes +} + +// Address returns a P2P endpoint address for the node. +func (n Node) AddressP2P(withID bool) string { + addr := fmt.Sprintf("%v:26656", n.IP.String()) + if withID { + addr = fmt.Sprintf("%x@%v", n.NetworkKey.PubKey().Address().Bytes(), addr) + } + return addr +} + +// Address returns an RPC endpoint address for the node. +func (n Node) AddressRPC() string { + return fmt.Sprintf("%v:%d", n.IP.String(), rpcPort) +} + +func (n Node) IsValidator() bool { + return n.SelfDelegation != 0 +} + +func (n Node) Client() (*http.HTTP, error) { + return http.New(fmt.Sprintf("http://127.0.0.1:%v", n.ProxyPort), "/websocket") +} + +func (n Node) Validate() error { + if len(n.Versions) == 0 { + return errors.New("at least one version is required") + } + if n.StartHeight < 0 { + return errors.New("start height must be non-negative") + } + if n.SelfDelegation < 0 { + return errors.New("self delegation must be non-negative") + } + return nil +} + +func (a Account) Validate() error { + if a.Tokens < 0 { + return errors.New("tokens must be non-negative") + } + return nil +} + +type keyGenerator struct { + random *rand.Rand +} + +func newKeyGenerator(seed int64) *keyGenerator { + return &keyGenerator{ + random: rand.New(rand.NewSource(seed)), //nolint:gosec + } +} + +func (g *keyGenerator) Generate(keyType string) crypto.PrivKey { + seed := make([]byte, ed25519.SeedSize) + + _, err := io.ReadFull(g.random, seed) + if err != nil { + panic(err) // this shouldn't happen + } + switch keyType { + case "secp256k1": + return secp256k1.GenPrivKeySecp256k1(seed) + case "", "ed25519": + return ed25519.GenPrivKeyFromSecret(seed) + default: + panic("KeyType not supported") // should not make it this far + } +} + +type ipGenerator struct { + network *net.IPNet + nextIP net.IP +} + +func newIPGenerator(network *net.IPNet) *ipGenerator { + nextIP := make([]byte, len(network.IP)) + copy(nextIP, network.IP) + gen := &ipGenerator{network: network, nextIP: nextIP} + // Skip network and gateway addresses + gen.Next() + gen.Next() + return gen +} + +func (g *ipGenerator) Network() *net.IPNet { + n := &net.IPNet{ + IP: make([]byte, len(g.network.IP)), + Mask: make([]byte, len(g.network.Mask)), + } + copy(n.IP, g.network.IP) + copy(n.Mask, g.network.Mask) + return n +} + +func (g *ipGenerator) Next() net.IP { + ip := make([]byte, len(g.nextIP)) + copy(ip, g.nextIP) + for i := len(g.nextIP) - 1; i >= 0; i-- { + g.nextIP[i]++ + if g.nextIP[i] != 0 { + break + } + } + return ip +}