diff --git a/rafttest/datadriven_port.go b/rafttest/datadriven_port.go new file mode 100644 index 00000000..ba99c662 --- /dev/null +++ b/rafttest/datadriven_port.go @@ -0,0 +1,71 @@ +package rafttest + +import ( + "fmt" + "strconv" + + "github.com/cockroachdb/datadriven" +) + +// scanArgs is a copy-paste from datadriven. TODO +func scanArgs(args []datadriven.CmdArg, key string, dests ...interface{}) error { + var arg datadriven.CmdArg + for i := range args { + if args[i].Key == key { + arg = args[i] + break + } + } + if arg.Key == "" { + return fmt.Errorf("missing argument: %s", key) + } + if len(dests) != len(arg.Vals) { + return fmt.Errorf("%s: got %d destinations, but %d values", arg.Key, len(dests), len(arg.Vals)) + } + + for i := range dests { + if err := scanErr(arg, i, dests[i]); err != nil { + return fmt.Errorf("%s: failed to scan argument %d: %v", arg.Key, i, err) + } + } + return nil +} + +// scanErr is like Scan but returns an error rather than taking a testing.T to fatal. +func scanErr(arg datadriven.CmdArg, i int, dest interface{}) error { + if i < 0 || i >= len(arg.Vals) { + return fmt.Errorf("cannot scan index %d of key %s", i, arg.Key) + } + val := arg.Vals[i] + switch dest := dest.(type) { + case *string: + *dest = val + case *int: + n, err := strconv.ParseInt(val, 10, 64) + if err != nil { + return err + } + *dest = int(n) // assume 64bit ints + case *int64: + n, err := strconv.ParseInt(val, 10, 64) + if err != nil { + return err + } + *dest = n + case *uint64: + n, err := strconv.ParseUint(val, 10, 64) + if err != nil { + return err + } + *dest = n + case *bool: + b, err := strconv.ParseBool(val) + if err != nil { + return err + } + *dest = b + default: + return fmt.Errorf("unsupported type %T for destination #%d (might be easy to add it)", dest, i+1) + } + return nil +} diff --git a/rafttest/interaction_env_handler.go b/rafttest/interaction_env_handler.go index 2ffec2b4..87a9bb62 100644 --- a/rafttest/interaction_env_handler.go +++ b/rafttest/interaction_env_handler.go @@ -17,6 +17,7 @@ package rafttest import ( "fmt" "strconv" + "strings" "testing" "github.com/cockroachdb/datadriven" @@ -30,6 +31,81 @@ func (env *InteractionEnv) Handle(t *testing.T, d datadriven.TestData) string { env.Output.Reset() var err error switch d.Cmd { + case "propose-conf-change": + // Propose a configuration change, or transition out of a previously + // proposed joint configuration change that requested explicit + // transitions. When adding nodes, this command can be used to + // logically add nodes to the configuration, but add-nodes is needed + // to "create" the nodes. + // + // propose-conf-change node_id [v1=] [transition=] + // command string + // See ConfChangesFromString for command string format. + // Arguments are: + // node_id - the node proposing the configuration change. + // v1 - make one change at a time, false by default. + // transition - "auto" (the default), "explicit" or "implicit". + // Example: + // + // propose-conf-change 1 transition=explicit + // v1 v3 l4 r5 + // + // Example: + // + // propose-conf-change 2 v1=true + // v5 + err = env.handleProposeConfChange(t, d) + default: + err = env.handleMultiCommand(t, d) + } + // NB: the highest log level suppresses all output, including that of the + // handlers. This comes in useful during setup which can be chatty. + // However, errors are always logged. + if err != nil { + if env.Output.Quiet() { + return err.Error() + } + env.Output.WriteString(err.Error()) + } + if env.Output.Len() == 0 { + return "ok" + } + return env.Output.String() +} + +func (env *InteractionEnv) handleMultiCommand(t *testing.T, d datadriven.TestData) error { + err := env.handleSingle(t, d.Cmd, d.CmdArgs) + if err != nil { + return err + } + + commands := d.Input + for len(commands) != 0 { + command := commands + next := strings.IndexByte(commands, '\n') + if next == -1 { + commands = "" + } else { + command, commands = commands[:next], commands[next+1:] + } + cmd, args, err := datadriven.ParseLine(command) + if err != nil { + t.Fatalf("could not parse the command line %q: %v", command, err) + } + + if err := env.handleSingle(t, cmd, args); err != nil { + if env.Output.Quiet() { + return err + } + env.Output.WriteString(err.Error()) + } + } + + return nil +} + +func (env *InteractionEnv) handleSingle(t *testing.T, cmd string, args []datadriven.CmdArg) error { + switch cmd { case "_breakpoint": // This is a helper case to attach a debugger to when a problem needs // to be investigated in a longer test file. In such a case, add the @@ -44,39 +120,39 @@ func (env *InteractionEnv) Handle(t *testing.T, d datadriven.TestData) string { // Example: // // add-nodes voters=(1 2 3) learners=(4 5) index=2 content=foo async-storage-writes=true - err = env.handleAddNodes(t, d) + return env.handleAddNodes(t, args) case "campaign": // Example: // // campaign - err = env.handleCampaign(t, d) + return env.handleCampaign(t, args) case "compact": // Example: // // compact - err = env.handleCompact(t, d) + return env.handleCompact(t, args) case "deliver-msgs": // Deliver the messages for a given recipient. // // Example: // // deliver-msgs type=MsgApp drop=(2,3) - err = env.handleDeliverMsgs(t, d) + return env.handleDeliverMsgs(t, args) case "process-ready": // Example: // // process-ready 3 - err = env.handleProcessReady(t, d) + return env.handleProcessReady(t, args) case "process-append-thread": // Example: // // process-append-thread 3 - err = env.handleProcessAppendThread(t, d) + return env.handleProcessAppendThread(t, args) case "process-apply-thread": // Example: // // process-apply-thread 3 - err = env.handleProcessApplyThread(t, d) + return env.handleProcessApplyThread(t, args) case "log-level": // Set the log level. NONE disables all output, including from the test // harness (except errors). @@ -84,19 +160,19 @@ func (env *InteractionEnv) Handle(t *testing.T, d datadriven.TestData) string { // Example: // // log-level WARN - err = env.handleLogLevel(d) + return env.handleLogLevel(args) case "raft-log": // Print the Raft log. // // Example: // // raft-log 3 - err = env.handleRaftLog(t, d) + return env.handleRaftLog(t, args) case "raft-state": // Print Raft state of all nodes (whether the node is leading, // following, etc.). The information for node n is based on // n's view. - err = env.handleRaftState() + return env.handleRaftState() case "set-randomized-election-timeout": // Set the randomized election timeout for the given node. Will be reset // again when the node changes state. @@ -104,7 +180,7 @@ func (env *InteractionEnv) Handle(t *testing.T, d datadriven.TestData) string { // Example: // // set-randomized-election-timeout 1 timeout=5 - err = env.handleSetRandomizedElectionTimeout(t, d) + return env.handleSetRandomizedElectionTimeout(t, args) case "stabilize": // Deliver messages to and run process-ready on the set of IDs until // no more work is to be done. If no ids are given, all nodes are used. @@ -112,14 +188,14 @@ func (env *InteractionEnv) Handle(t *testing.T, d datadriven.TestData) string { // Example: // // stabilize 1 4 - err = env.handleStabilize(t, d) + return env.handleStabilize(t, args) case "status": // Print Raft status. // // Example: // // status 5 - err = env.handleStatus(t, d) + return env.handleStatus(t, args) case "tick-election": // Tick an election timeout interval for the given node (but beware the // randomized timeout). @@ -127,111 +203,74 @@ func (env *InteractionEnv) Handle(t *testing.T, d datadriven.TestData) string { // Example: // // tick-election 3 - err = env.handleTickElection(t, d) + return env.handleTickElection(t, args) case "tick-heartbeat": // Tick a heartbeat interval. // // Example: // // tick-heartbeat 3 - err = env.handleTickHeartbeat(t, d) + return env.handleTickHeartbeat(t, args) case "transfer-leadership": // Transfer the Raft leader. // // Example: // // transfer-leadership from=1 to=4 - err = env.handleTransferLeadership(t, d) + return env.handleTransferLeadership(t, args) case "forget-leader": // Forgets the current leader of the given node. // // Example: // // forget-leader 1 - err = env.handleForgetLeader(t, d) + return env.handleForgetLeader(t, args) case "send-snapshot": // Sends a snapshot to a node. Takes the source and destination node. // The message will be queued, but not delivered automatically. // // Example: send-snapshot 1 3 - env.handleSendSnapshot(t, d) + return env.handleSendSnapshot(t, args) case "propose": // Propose an entry. // // Example: // // propose 1 foo - err = env.handlePropose(t, d) - case "propose-conf-change": - // Propose a configuration change, or transition out of a previously - // proposed joint configuration change that requested explicit - // transitions. When adding nodes, this command can be used to - // logically add nodes to the configuration, but add-nodes is needed - // to "create" the nodes. - // - // propose-conf-change node_id [v1=] [transition=] - // command string - // See ConfChangesFromString for command string format. - // Arguments are: - // node_id - the node proposing the configuration change. - // v1 - make one change at a time, false by default. - // transition - "auto" (the default), "explicit" or "implicit". - // Example: - // - // propose-conf-change 1 transition=explicit - // v1 v3 l4 r5 - // - // Example: - // - // propose-conf-change 2 v1=true - // v5 - err = env.handleProposeConfChange(t, d) + return env.handlePropose(t, args) case "report-unreachable": // Calls <1st>.ReportUnreachable(<2nd>). // // Example: // report-unreachable 1 2 - err = env.handleReportUnreachable(t, d) + return env.handleReportUnreachable(t, args) default: - err = fmt.Errorf("unknown command") - } - // NB: the highest log level suppresses all output, including that of the - // handlers. This comes in useful during setup which can be chatty. - // However, errors are always logged. - if err != nil { - if env.Output.Quiet() { - return err.Error() - } - env.Output.WriteString(err.Error()) } - if env.Output.Len() == 0 { - return "ok" - } - return env.Output.String() + return fmt.Errorf("unknown command") } -func firstAsInt(t *testing.T, d datadriven.TestData) int { +func firstAsInt(t *testing.T, args []datadriven.CmdArg) int { t.Helper() - n, err := strconv.Atoi(d.CmdArgs[0].Key) + n, err := strconv.Atoi(args[0].Key) if err != nil { t.Fatal(err) } return n } -func firstAsNodeIdx(t *testing.T, d datadriven.TestData) int { +func firstAsNodeIdx(t *testing.T, args []datadriven.CmdArg) int { t.Helper() - n := firstAsInt(t, d) + n := firstAsInt(t, args) return n - 1 } -func nodeIdxs(t *testing.T, d datadriven.TestData) []int { +func nodeIdxs(t *testing.T, args []datadriven.CmdArg) []int { var ints []int - for i := 0; i < len(d.CmdArgs); i++ { - if len(d.CmdArgs[i].Vals) != 0 { + for i := 0; i < len(args); i++ { + if len(args[i].Vals) != 0 { continue } - n, err := strconv.Atoi(d.CmdArgs[i].Key) + n, err := strconv.Atoi(args[i].Key) if err != nil { t.Fatal(err) } diff --git a/rafttest/interaction_env_handler_add_nodes.go b/rafttest/interaction_env_handler_add_nodes.go index e68a295f..346b087c 100644 --- a/rafttest/interaction_env_handler_add_nodes.go +++ b/rafttest/interaction_env_handler_add_nodes.go @@ -26,11 +26,11 @@ import ( pb "go.etcd.io/raft/v3/raftpb" ) -func (env *InteractionEnv) handleAddNodes(t *testing.T, d datadriven.TestData) error { - n := firstAsInt(t, d) +func (env *InteractionEnv) handleAddNodes(t *testing.T, args []datadriven.CmdArg) error { + n := firstAsInt(t, args) var snap pb.Snapshot cfg := raftConfigStub() - for _, arg := range d.CmdArgs[1:] { + for _, arg := range args[1:] { for i := range arg.Vals { switch arg.Key { case "voters": diff --git a/rafttest/interaction_env_handler_campaign.go b/rafttest/interaction_env_handler_campaign.go index 8b6ccbc1..d9649706 100644 --- a/rafttest/interaction_env_handler_campaign.go +++ b/rafttest/interaction_env_handler_campaign.go @@ -20,8 +20,8 @@ import ( "github.com/cockroachdb/datadriven" ) -func (env *InteractionEnv) handleCampaign(t *testing.T, d datadriven.TestData) error { - idx := firstAsNodeIdx(t, d) +func (env *InteractionEnv) handleCampaign(t *testing.T, args []datadriven.CmdArg) error { + idx := firstAsNodeIdx(t, args) return env.Campaign(idx) } diff --git a/rafttest/interaction_env_handler_compact.go b/rafttest/interaction_env_handler_compact.go index 25fa1d22..bb58e8e7 100644 --- a/rafttest/interaction_env_handler_compact.go +++ b/rafttest/interaction_env_handler_compact.go @@ -21,9 +21,9 @@ import ( "github.com/cockroachdb/datadriven" ) -func (env *InteractionEnv) handleCompact(t *testing.T, d datadriven.TestData) error { - idx := firstAsNodeIdx(t, d) - newFirstIndex, err := strconv.ParseUint(d.CmdArgs[1].Key, 10, 64) +func (env *InteractionEnv) handleCompact(t *testing.T, args []datadriven.CmdArg) error { + idx := firstAsNodeIdx(t, args) + newFirstIndex, err := strconv.ParseUint(args[1].Key, 10, 64) if err != nil { return err } diff --git a/rafttest/interaction_env_handler_deliver_msgs.go b/rafttest/interaction_env_handler_deliver_msgs.go index af8534cb..90e06c18 100644 --- a/rafttest/interaction_env_handler_deliver_msgs.go +++ b/rafttest/interaction_env_handler_deliver_msgs.go @@ -25,10 +25,10 @@ import ( "go.etcd.io/raft/v3/raftpb" ) -func (env *InteractionEnv) handleDeliverMsgs(t *testing.T, d datadriven.TestData) error { +func (env *InteractionEnv) handleDeliverMsgs(t *testing.T, args []datadriven.CmdArg) error { var typ raftpb.MessageType = -1 var rs []Recipient - for _, arg := range d.CmdArgs { + for _, arg := range args { if len(arg.Vals) == 0 { id, err := strconv.ParseUint(arg.Key, 10, 64) if err != nil { diff --git a/rafttest/interaction_env_handler_forget_leader.go b/rafttest/interaction_env_handler_forget_leader.go index b239db75..b2ae7462 100644 --- a/rafttest/interaction_env_handler_forget_leader.go +++ b/rafttest/interaction_env_handler_forget_leader.go @@ -20,8 +20,8 @@ import ( "github.com/cockroachdb/datadriven" ) -func (env *InteractionEnv) handleForgetLeader(t *testing.T, d datadriven.TestData) error { - idx := firstAsNodeIdx(t, d) +func (env *InteractionEnv) handleForgetLeader(t *testing.T, args []datadriven.CmdArg) error { + idx := firstAsNodeIdx(t, args) env.ForgetLeader(idx) return nil } diff --git a/rafttest/interaction_env_handler_log_level.go b/rafttest/interaction_env_handler_log_level.go index c517a4d7..25286efb 100644 --- a/rafttest/interaction_env_handler_log_level.go +++ b/rafttest/interaction_env_handler_log_level.go @@ -21,8 +21,8 @@ import ( "github.com/cockroachdb/datadriven" ) -func (env *InteractionEnv) handleLogLevel(d datadriven.TestData) error { - return env.LogLevel(d.CmdArgs[0].Key) +func (env *InteractionEnv) handleLogLevel(args []datadriven.CmdArg) error { + return env.LogLevel(args[0].Key) } func (env *InteractionEnv) LogLevel(name string) error { diff --git a/rafttest/interaction_env_handler_process_append_thread.go b/rafttest/interaction_env_handler_process_append_thread.go index 41363210..6755bd46 100644 --- a/rafttest/interaction_env_handler_process_append_thread.go +++ b/rafttest/interaction_env_handler_process_append_thread.go @@ -25,8 +25,8 @@ import ( "go.etcd.io/raft/v3/raftpb" ) -func (env *InteractionEnv) handleProcessAppendThread(t *testing.T, d datadriven.TestData) error { - idxs := nodeIdxs(t, d) +func (env *InteractionEnv) handleProcessAppendThread(t *testing.T, args []datadriven.CmdArg) error { + idxs := nodeIdxs(t, args) for _, idx := range idxs { var err error if len(idxs) > 1 { diff --git a/rafttest/interaction_env_handler_process_apply_thread.go b/rafttest/interaction_env_handler_process_apply_thread.go index d21317e0..1b87da85 100644 --- a/rafttest/interaction_env_handler_process_apply_thread.go +++ b/rafttest/interaction_env_handler_process_apply_thread.go @@ -24,8 +24,8 @@ import ( "go.etcd.io/raft/v3/raftpb" ) -func (env *InteractionEnv) handleProcessApplyThread(t *testing.T, d datadriven.TestData) error { - idxs := nodeIdxs(t, d) +func (env *InteractionEnv) handleProcessApplyThread(t *testing.T, args []datadriven.CmdArg) error { + idxs := nodeIdxs(t, args) for _, idx := range idxs { var err error if len(idxs) > 1 { diff --git a/rafttest/interaction_env_handler_process_ready.go b/rafttest/interaction_env_handler_process_ready.go index e72d3d9d..64c89d42 100644 --- a/rafttest/interaction_env_handler_process_ready.go +++ b/rafttest/interaction_env_handler_process_ready.go @@ -24,8 +24,8 @@ import ( "go.etcd.io/raft/v3/raftpb" ) -func (env *InteractionEnv) handleProcessReady(t *testing.T, d datadriven.TestData) error { - idxs := nodeIdxs(t, d) +func (env *InteractionEnv) handleProcessReady(t *testing.T, args []datadriven.CmdArg) error { + idxs := nodeIdxs(t, args) for _, idx := range idxs { var err error if len(idxs) > 1 { diff --git a/rafttest/interaction_env_handler_propose.go b/rafttest/interaction_env_handler_propose.go index 7e883234..0a2353ec 100644 --- a/rafttest/interaction_env_handler_propose.go +++ b/rafttest/interaction_env_handler_propose.go @@ -20,12 +20,12 @@ import ( "github.com/cockroachdb/datadriven" ) -func (env *InteractionEnv) handlePropose(t *testing.T, d datadriven.TestData) error { - idx := firstAsNodeIdx(t, d) - if len(d.CmdArgs) != 2 || len(d.CmdArgs[1].Vals) > 0 { - t.Fatalf("expected exactly one key with no vals: %+v", d.CmdArgs[1:]) +func (env *InteractionEnv) handlePropose(t *testing.T, args []datadriven.CmdArg) error { + idx := firstAsNodeIdx(t, args) + if len(args) != 2 || len(args[1].Vals) > 0 { + t.Fatalf("expected exactly one key with no vals: %+v", args[1:]) } - return env.Propose(idx, []byte(d.CmdArgs[1].Key)) + return env.Propose(idx, []byte(args[1].Key)) } // Propose a regular entry. diff --git a/rafttest/interaction_env_handler_propose_conf_change.go b/rafttest/interaction_env_handler_propose_conf_change.go index 531e553e..eb3682e3 100644 --- a/rafttest/interaction_env_handler_propose_conf_change.go +++ b/rafttest/interaction_env_handler_propose_conf_change.go @@ -25,7 +25,7 @@ import ( ) func (env *InteractionEnv) handleProposeConfChange(t *testing.T, d datadriven.TestData) error { - idx := firstAsNodeIdx(t, d) + idx := firstAsNodeIdx(t, d.CmdArgs) var v1 bool transition := raftpb.ConfChangeTransitionAuto for _, arg := range d.CmdArgs[1:] { diff --git a/rafttest/interaction_env_handler_raft_log.go b/rafttest/interaction_env_handler_raft_log.go index d4bc1b13..fee3f61b 100644 --- a/rafttest/interaction_env_handler_raft_log.go +++ b/rafttest/interaction_env_handler_raft_log.go @@ -24,8 +24,8 @@ import ( "go.etcd.io/raft/v3" ) -func (env *InteractionEnv) handleRaftLog(t *testing.T, d datadriven.TestData) error { - idx := firstAsNodeIdx(t, d) +func (env *InteractionEnv) handleRaftLog(t *testing.T, args []datadriven.CmdArg) error { + idx := firstAsNodeIdx(t, args) return env.RaftLog(idx) } diff --git a/rafttest/interaction_env_handler_report_unreachable.go b/rafttest/interaction_env_handler_report_unreachable.go index dce46ba2..5acb13d8 100644 --- a/rafttest/interaction_env_handler_report_unreachable.go +++ b/rafttest/interaction_env_handler_report_unreachable.go @@ -20,8 +20,8 @@ import ( "github.com/cockroachdb/datadriven" ) -func (env *InteractionEnv) handleReportUnreachable(t *testing.T, d datadriven.TestData) error { - sl := nodeIdxs(t, d) +func (env *InteractionEnv) handleReportUnreachable(t *testing.T, args []datadriven.CmdArg) error { + sl := nodeIdxs(t, args) if len(sl) != 2 { return errors.New("must specify exactly two node indexes: node on which to report, and reported node") } diff --git a/rafttest/interaction_env_handler_send_snapshot.go b/rafttest/interaction_env_handler_send_snapshot.go index c91d4cc8..e68926af 100644 --- a/rafttest/interaction_env_handler_send_snapshot.go +++ b/rafttest/interaction_env_handler_send_snapshot.go @@ -24,8 +24,8 @@ import ( "go.etcd.io/raft/v3/raftpb" ) -func (env *InteractionEnv) handleSendSnapshot(t *testing.T, d datadriven.TestData) error { - idxs := nodeIdxs(t, d) +func (env *InteractionEnv) handleSendSnapshot(t *testing.T, args []datadriven.CmdArg) error { + idxs := nodeIdxs(t, args) require.Len(t, idxs, 2) return env.SendSnapshot(idxs[0], idxs[1]) } diff --git a/rafttest/interaction_env_handler_set_randomized_election_timeout.go b/rafttest/interaction_env_handler_set_randomized_election_timeout.go index 36f0cd2d..cfdcb29c 100644 --- a/rafttest/interaction_env_handler_set_randomized_election_timeout.go +++ b/rafttest/interaction_env_handler_set_randomized_election_timeout.go @@ -22,11 +22,13 @@ import ( ) func (env *InteractionEnv) handleSetRandomizedElectionTimeout( - t *testing.T, d datadriven.TestData, + t *testing.T, args []datadriven.CmdArg, ) error { - idx := firstAsNodeIdx(t, d) + idx := firstAsNodeIdx(t, args) var timeout int - d.ScanArgs(t, "timeout", &timeout) + if err := scanArgs(args, "timeout", &timeout); err != nil { + t.Fatal(err) // FIXME: TestData.Fatalf(t, err) + } require.NotZero(t, timeout) env.Options.SetRandomizedElectionTimeout(env.Nodes[idx].RawNode, timeout) diff --git a/rafttest/interaction_env_handler_stabilize.go b/rafttest/interaction_env_handler_stabilize.go index 56ae2e1e..070a8784 100644 --- a/rafttest/interaction_env_handler_stabilize.go +++ b/rafttest/interaction_env_handler_stabilize.go @@ -24,9 +24,9 @@ import ( "go.etcd.io/raft/v3/raftpb" ) -func (env *InteractionEnv) handleStabilize(t *testing.T, d datadriven.TestData) error { - idxs := nodeIdxs(t, d) // skips key=value args - for _, arg := range d.CmdArgs { +func (env *InteractionEnv) handleStabilize(t *testing.T, args []datadriven.CmdArg) error { + idxs := nodeIdxs(t, args) // skips key=value args + for _, arg := range args { for i := range arg.Vals { switch arg.Key { case "log-level": diff --git a/rafttest/interaction_env_handler_status.go b/rafttest/interaction_env_handler_status.go index 40bb7d34..72660a07 100644 --- a/rafttest/interaction_env_handler_status.go +++ b/rafttest/interaction_env_handler_status.go @@ -23,8 +23,8 @@ import ( "go.etcd.io/raft/v3/tracker" ) -func (env *InteractionEnv) handleStatus(t *testing.T, d datadriven.TestData) error { - idx := firstAsNodeIdx(t, d) +func (env *InteractionEnv) handleStatus(t *testing.T, args []datadriven.CmdArg) error { + idx := firstAsNodeIdx(t, args) return env.Status(idx) } diff --git a/rafttest/interaction_env_handler_tick.go b/rafttest/interaction_env_handler_tick.go index 10c9ba21..3345e5ce 100644 --- a/rafttest/interaction_env_handler_tick.go +++ b/rafttest/interaction_env_handler_tick.go @@ -20,13 +20,13 @@ import ( "github.com/cockroachdb/datadriven" ) -func (env *InteractionEnv) handleTickElection(t *testing.T, d datadriven.TestData) error { - idx := firstAsNodeIdx(t, d) +func (env *InteractionEnv) handleTickElection(t *testing.T, args []datadriven.CmdArg) error { + idx := firstAsNodeIdx(t, args) return env.Tick(idx, env.Nodes[idx].Config.ElectionTick) } -func (env *InteractionEnv) handleTickHeartbeat(t *testing.T, d datadriven.TestData) error { - idx := firstAsNodeIdx(t, d) +func (env *InteractionEnv) handleTickHeartbeat(t *testing.T, args []datadriven.CmdArg) error { + idx := firstAsNodeIdx(t, args) return env.Tick(idx, env.Nodes[idx].Config.HeartbeatTick) } diff --git a/rafttest/interaction_env_handler_transfer_leadership.go b/rafttest/interaction_env_handler_transfer_leadership.go index dc7f366d..12ea1dde 100644 --- a/rafttest/interaction_env_handler_transfer_leadership.go +++ b/rafttest/interaction_env_handler_transfer_leadership.go @@ -20,10 +20,14 @@ import ( "github.com/cockroachdb/datadriven" ) -func (env *InteractionEnv) handleTransferLeadership(t *testing.T, d datadriven.TestData) error { +func (env *InteractionEnv) handleTransferLeadership(t *testing.T, args []datadriven.CmdArg) error { var from, to uint64 - d.ScanArgs(t, "from", &from) - d.ScanArgs(t, "to", &to) + if err := scanArgs(args, "from", &from); err != nil { + t.Fatal(err) // FIXME: TestData.Fatalf(t, err) + } + if err := scanArgs(args, "to", &to); err != nil { + t.Fatal(err) // FIXME: TestData.Fatalf(t, err) + } if from == 0 || from > uint64(len(env.Nodes)) { t.Fatalf(`expected valid "from" argument`) } diff --git a/testdata/async_storage_writes.txt b/testdata/async_storage_writes.txt index 52f37092..5d0b3f6f 100644 --- a/testdata/async_storage_writes.txt +++ b/testdata/async_storage_writes.txt @@ -200,9 +200,6 @@ stabilize ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] propose 1 prop_1 ----- -ok - process-ready 1 2 3 ---- > 1 handling Ready @@ -241,9 +238,6 @@ process-ready 1 2 3 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[3->1 MsgAppResp Term:1 Log:0/12, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/12] propose 1 prop_2 ----- -ok - process-ready 1 2 3 ---- > 1 handling Ready @@ -312,9 +306,6 @@ AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/12 propose 1 prop_3 ----- -ok - process-ready 1 2 3 ---- > 1 handling Ready @@ -399,9 +390,6 @@ AppendThread->2 MsgStorageAppendResp Term:1 Log:1/13 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/13 propose 1 prop_4 ----- -ok - process-ready 1 2 3 ---- > 1 handling Ready diff --git a/testdata/async_storage_writes_append_aba_race.txt b/testdata/async_storage_writes_append_aba_race.txt index 4b68330f..14e2417f 100644 --- a/testdata/async_storage_writes_append_aba_race.txt +++ b/testdata/async_storage_writes_append_aba_race.txt @@ -2,34 +2,18 @@ # newStorageAppendRespMsg, exercising a few interesting interactions # between asynchronous storage writes, term changes, and log truncation. +# Step 1: node 2 is the leader. log-level none ----- -ok - add-nodes 7 voters=(1,2,3,4,5,6,7) index=10 async-storage-writes=true ----- -ok - -# Step 1: node 2 is the leader. - campaign 2 ----- -ok - stabilize ---- ok -log-level info ----- -ok - # Step 2: node 2 proposes some log entries but only node 1 receives these entries. +log-level info propose 2 init_prop ----- -ok - process-ready 2 ---- Ready MustSync=true: @@ -339,9 +323,6 @@ dropped: 4->1 MsgVote Term:3 Log:1/11 dropped: 4->1 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] tick-heartbeat 4 ----- -ok - process-ready 4 ---- Ready MustSync=false: diff --git a/testdata/campaign_learner_must_vote.txt b/testdata/campaign_learner_must_vote.txt index 85617df3..5e5c0bca 100644 --- a/testdata/campaign_learner_must_vote.txt +++ b/testdata/campaign_learner_must_vote.txt @@ -8,21 +8,10 @@ # See: # https://github.com/etcd-io/etcd/pull/10998 -# Turn output off during boilerplate. +# Bootstrap three nodes, and elect node 1 as leader. log-level none ----- -ok - -# Bootstrap three nodes. add-nodes 3 voters=(1,2) learners=(3) index=2 ----- -ok - -# n1 gets to be leader. campaign 1 ----- -ok - stabilize ---- ok @@ -47,9 +36,6 @@ ok # We now pretend that n1 is dead, and n2 is trying to become leader. log-level debug ----- -ok - campaign 2 ---- INFO 2 is starting a new election at term 1 diff --git a/testdata/checkquorum.txt b/testdata/checkquorum.txt index b25c1e63..df14a1b0 100644 --- a/testdata/checkquorum.txt +++ b/testdata/checkquorum.txt @@ -5,35 +5,21 @@ # thesis this is part of PreVote, but etcd/raft enables this via CheckQuorum. log-level none ----- -ok - add-nodes 3 voters=(1,2,3) index=10 checkquorum=true ----- -ok - campaign 1 ----- -ok - stabilize ----- -ok - log-level debug ---- ok # Campaigning will fail when there is an active leader. campaign 2 +stabilize ---- INFO 2 is starting a new election at term 1 INFO 2 became candidate at term 2 INFO 2 [logterm: 1, index: 11] sent MsgVote request to 1 at term 2 INFO 2 [logterm: 1, index: 11] sent MsgVote request to 3 at term 2 - -stabilize ----- > 2 handling Ready Ready MustSync=true: Lead:0 State:StateCandidate @@ -54,9 +40,6 @@ stabilize # tick 2 election timeouts, since the followers were active in the current # interval (see messages above). tick-election 1 ----- -ok - tick-election 1 ---- WARN 1 stepped down to follower since quorum is not active @@ -149,15 +132,12 @@ Messages: INFO 2 received MsgVoteResp from 2 at term 3 INFO 2 has received 1 MsgVoteResp votes and 0 vote rejections -deliver-msgs 1 +deliver-msgs 1 3 ---- 2->1 MsgVote Term:3 Log:1/11 INFO 1 [term: 2] received a MsgVote message with higher term from 2 [term: 3] INFO 1 became follower at term 3 INFO 1 [logterm: 1, index: 11, vote: 0] cast MsgVote for 2 [logterm: 1, index: 11] at term 3 - -deliver-msgs 3 ----- 2->3 MsgVote Term:3 Log:1/11 INFO 3 [logterm: 1, index: 11, vote: 1] ignored MsgVote from 2 [logterm: 1, index: 11] at term 1: lease is not expired (remaining ticks: 3) diff --git a/testdata/confchange_disable_validation.txt b/testdata/confchange_disable_validation.txt index 1a2bc4fd..9dab28ca 100644 --- a/testdata/confchange_disable_validation.txt +++ b/testdata/confchange_disable_validation.txt @@ -8,20 +8,16 @@ # configuration change is proposed. That configuration change is accepted into # the log since due to DisableConfChangeValidation=true. add-nodes 1 voters=(1) index=2 max-committed-size-per-ready=1 disable-conf-change-validation=true +campaign 1 +log-level none +stabilize ---- INFO 1 switched to configuration voters=(1) INFO 1 became follower at term 0 INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] - -campaign 1 ----- INFO 1 is starting a new election at term 0 INFO 1 became candidate at term 1 -stabilize log-level=none ----- -ok - # Dummy entry. propose 1 foo ---- @@ -35,18 +31,12 @@ ok # Entries both get appended. process-ready 1 ---- -Ready MustSync=true: -Entries: -1/4 EntryNormal "foo" -1/5 EntryConfChangeV2 l2 l3 +ok # Dummy entry comes up for application. process-ready 1 ---- -Ready MustSync=false: -HardState Term:1 Vote:1 Commit:5 -CommittedEntries: -1/4 EntryNormal "foo" +ok # Propose new config change. Note how it isn't rejected, # which is due to DisableConfChangeValidation=true. @@ -58,19 +48,4 @@ ok # second one gets committed and also applies. stabilize ---- -> 1 handling Ready - Ready MustSync=true: - Entries: - 1/6 EntryConfChangeV2 - CommittedEntries: - 1/5 EntryConfChangeV2 l2 l3 - INFO 1 switched to configuration voters=(1)&&(1) learners=(2 3) -> 1 handling Ready - Ready MustSync=false: - HardState Term:1 Vote:1 Commit:6 - CommittedEntries: - 1/6 EntryConfChangeV2 - Messages: - 1->2 MsgApp Term:1 Log:1/5 Commit:5 Entries:[1/6 EntryConfChangeV2] - 1->3 MsgApp Term:1 Log:1/5 Commit:5 Entries:[1/6 EntryConfChangeV2] - INFO 1 switched to configuration voters=(1) learners=(2 3) +ok diff --git a/testdata/confchange_v1_add_single.txt b/testdata/confchange_v1_add_single.txt index 0419d28d..89d9006f 100644 --- a/testdata/confchange_v1_add_single.txt +++ b/testdata/confchange_v1_add_single.txt @@ -2,13 +2,11 @@ # Bootstrap n1. add-nodes 1 voters=(1) index=2 +campaign 1 ---- INFO 1 switched to configuration voters=(1) INFO 1 became follower at term 0 INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] - -campaign 1 ----- INFO 1 is starting a new election at term 0 INFO 1 became candidate at term 1 diff --git a/testdata/confchange_v1_remove_leader.txt b/testdata/confchange_v1_remove_leader.txt index cc91508a..badf9c66 100644 --- a/testdata/confchange_v1_remove_leader.txt +++ b/testdata/confchange_v1_remove_leader.txt @@ -1,26 +1,14 @@ -# We'll turn this back on after the boilerplate. -log-level none ----- -ok - # Run a V1 membership change that removes the leader. + # Bootstrap n1, n2, n3. +log-level none add-nodes 3 voters=(1,2,3) index=2 ----- -ok - campaign 1 ----- -ok - stabilize ---- ok log-level debug ----- -ok - raft-state ---- 1: StateLeader (Voter) Term:1 Lead:1 diff --git a/testdata/confchange_v1_remove_leader_stepdown.txt b/testdata/confchange_v1_remove_leader_stepdown.txt index fe397650..289508b5 100644 --- a/testdata/confchange_v1_remove_leader_stepdown.txt +++ b/testdata/confchange_v1_remove_leader_stepdown.txt @@ -1,27 +1,15 @@ -# We'll turn this back on after the boilerplate. -log-level none ----- -ok - # Run a V1 membership change that removes the leader, asking it # to step down on removal. + # Bootstrap n1, n2, n3. +log-level none add-nodes 3 voters=(1,2,3) index=2 step-down-on-removal=true ----- -ok - campaign 1 ----- -ok - stabilize ---- ok log-level debug ----- -ok - raft-state ---- 1: StateLeader (Voter) Term:1 Lead:1 diff --git a/testdata/confchange_v2_add_double_auto.txt b/testdata/confchange_v2_add_double_auto.txt index 5cf4f24c..ff70d3a6 100644 --- a/testdata/confchange_v2_add_double_auto.txt +++ b/testdata/confchange_v2_add_double_auto.txt @@ -4,13 +4,11 @@ # Bootstrap n1. add-nodes 1 voters=(1) index=2 +campaign 1 ---- INFO 1 switched to configuration voters=(1) INFO 1 became follower at term 0 INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] - -campaign 1 ----- INFO 1 is starting a new election at term 0 INFO 1 became candidate at term 1 @@ -250,9 +248,6 @@ stabilize 2 3 # be a bug in which these proposals would prompt the leader to transition out of # the same joint state multiple times, which would cause a panic. propose 1 foo ----- -ok - propose 1 bar ---- ok diff --git a/testdata/confchange_v2_add_double_implicit.txt b/testdata/confchange_v2_add_double_implicit.txt index 81b3f0de..2b1a0640 100644 --- a/testdata/confchange_v2_add_double_implicit.txt +++ b/testdata/confchange_v2_add_double_implicit.txt @@ -6,13 +6,11 @@ # Bootstrap n1. add-nodes 1 voters=(1) index=2 +campaign 1 ---- INFO 1 switched to configuration voters=(1) INFO 1 became follower at term 0 INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] - -campaign 1 ----- INFO 1 is starting a new election at term 0 INFO 1 became candidate at term 1 diff --git a/testdata/confchange_v2_add_single_auto.txt b/testdata/confchange_v2_add_single_auto.txt index fe127bf5..4f5c9f5b 100644 --- a/testdata/confchange_v2_add_single_auto.txt +++ b/testdata/confchange_v2_add_single_auto.txt @@ -4,13 +4,11 @@ # Bootstrap n1. add-nodes 1 voters=(1) index=2 +campaign 1 ---- INFO 1 switched to configuration voters=(1) INFO 1 became follower at term 0 INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] - -campaign 1 ----- INFO 1 is starting a new election at term 0 INFO 1 became candidate at term 1 diff --git a/testdata/confchange_v2_add_single_explicit.txt b/testdata/confchange_v2_add_single_explicit.txt index c51900f0..d511dc5a 100644 --- a/testdata/confchange_v2_add_single_explicit.txt +++ b/testdata/confchange_v2_add_single_explicit.txt @@ -4,13 +4,11 @@ # Bootstrap n1. add-nodes 1 voters=(1) index=2 +campaign 1 ---- INFO 1 switched to configuration voters=(1) INFO 1 became follower at term 0 INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] - -campaign 1 ----- INFO 1 is starting a new election at term 0 INFO 1 became candidate at term 1 diff --git a/testdata/confchange_v2_replace_leader.txt b/testdata/confchange_v2_replace_leader.txt index dfb09505..c1b9bc44 100644 --- a/testdata/confchange_v2_replace_leader.txt +++ b/testdata/confchange_v2_replace_leader.txt @@ -4,29 +4,15 @@ # while in the joint config. After the reconfiguration completes, we verify # that the removed leader cannot campaign to become leader. -# We'll turn this back on after the boilerplate. +# Bootstrap n1, n2, n3. Elect 1 as the leader. log-level none ----- -ok - -# Bootstrap n1, n2, n3. add-nodes 3 voters=(1,2,3) index=2 ----- -ok - -# n1 campaigns to become leader. campaign 1 ----- -ok - stabilize ---- ok log-level info ----- -ok - raft-state ---- 1: StateLeader (Voter) Term:1 Lead:1 diff --git a/testdata/confchange_v2_replace_leader_stepdown.txt b/testdata/confchange_v2_replace_leader_stepdown.txt index 62d01d23..902da3e9 100644 --- a/testdata/confchange_v2_replace_leader_stepdown.txt +++ b/testdata/confchange_v2_replace_leader_stepdown.txt @@ -5,29 +5,15 @@ # reconfiguration completes, we verify that the removed leader cannot campaign # to become leader. -# We'll turn this back on after the boilerplate. +# Bootstrap n1, n2, n3. Elect 1 as the leader. log-level none ----- -ok - -# Bootstrap n1, n2, n3. add-nodes 3 voters=(1,2,3) index=2 step-down-on-removal=true ----- -ok - -# n1 campaigns to become leader. campaign 1 ----- -ok - stabilize ---- ok log-level info ----- -ok - raft-state ---- 1: StateLeader (Voter) Term:1 Lead:1 diff --git a/testdata/forget_leader.txt b/testdata/forget_leader.txt index a674b346..a4574368 100644 --- a/testdata/forget_leader.txt +++ b/testdata/forget_leader.txt @@ -1,24 +1,12 @@ log-level none ----- -ok - add-nodes 4 voters=(1,2,3) learners=(4) index=10 ----- -ok - campaign 1 ----- -ok - stabilize ---- ok -log-level debug ----- -ok - # ForgetLeader is a noop on the leader. +log-level debug forget-leader 1 ---- ok @@ -68,9 +56,6 @@ raft-state # When receiving a heartbeat from the leader, they revert to followers. tick-heartbeat 1 ----- -ok - stabilize ---- > 1 handling Ready @@ -162,9 +147,6 @@ set-randomized-election-timeout 2 timeout=3 ok tick-heartbeat 2 ----- -ok - tick-heartbeat 2 ---- ok diff --git a/testdata/forget_leader_prevote_checkquorum.txt b/testdata/forget_leader_prevote_checkquorum.txt index 9b3b80ff..490724c6 100644 --- a/testdata/forget_leader_prevote_checkquorum.txt +++ b/testdata/forget_leader_prevote_checkquorum.txt @@ -3,27 +3,16 @@ # # Also tests that forgetting the leader still won't grant prevotes to a # replica that isn't up-to-date. -log-level none ----- -ok +log-level none add-nodes 3 voters=(1,2,3) index=10 prevote=true checkquorum=true ----- -ok - campaign 1 ----- -ok - stabilize ---- ok -log-level debug ----- -ok - # If 3 attempts to campaign, 2 rejects it because it has a leader. +log-level debug campaign 3 ---- INFO 3 is starting a new election at term 1 @@ -51,9 +40,6 @@ INFO 2 [logterm: 1, index: 11, vote: 1] ignored MsgPreVote from 3 [logterm: 1, i # Make 1 assert leadership over 3 again. tick-heartbeat 1 ----- -ok - stabilize ---- > 1 handling Ready @@ -98,14 +84,12 @@ raft-state 3: StateFollower (Voter) Term:1 Lead:1 campaign 3 +stabilize 3 ---- INFO 3 is starting a new election at term 1 INFO 3 became pre-candidate at term 1 INFO 3 [logterm: 1, index: 11] sent MsgPreVote request to 1 at term 1 INFO 3 [logterm: 1, index: 11] sent MsgPreVote request to 2 at term 1 - -stabilize 3 ----- > 3 handling Ready Ready MustSync=false: Lead:0 State:StatePreCandidate @@ -160,9 +144,6 @@ raft-state # Test that forgetting the leader still won't grant prevotes if the candidate # isn't up-to-date. We first replicate a proposal on 3 and 2. propose 3 prop_1 ----- -ok - stabilize 3 ---- > 3 handling Ready diff --git a/testdata/forget_leader_read_only_lease_based.txt b/testdata/forget_leader_read_only_lease_based.txt index 4d4bd188..3b1def46 100644 --- a/testdata/forget_leader_read_only_lease_based.txt +++ b/testdata/forget_leader_read_only_lease_based.txt @@ -1,24 +1,12 @@ log-level none ----- -ok - add-nodes 3 voters=(1,2,3) index=10 checkquorum=true read-only=lease-based ----- -ok - campaign 1 ----- -ok - stabilize ---- ok -log-level debug ----- -ok - # ForgetLeader fails with lease-based reads, as it's not safe. +log-level debug forget-leader 2 ---- ERROR ignoring MsgForgetLeader due to ReadOnlyLeaseBased diff --git a/testdata/heartbeat_resp_recovers_from_probing.txt b/testdata/heartbeat_resp_recovers_from_probing.txt index e606a155..9ee1076f 100644 --- a/testdata/heartbeat_resp_recovers_from_probing.txt +++ b/testdata/heartbeat_resp_recovers_from_probing.txt @@ -5,27 +5,14 @@ # StateReplicate for the follower. In other words, we don't end up in # a stable state with a fully caught up follower in StateProbe. -# Turn off output during the setup of the test. log-level none ----- -ok - add-nodes 3 voters=(1,2,3) index=10 ----- -ok - campaign 1 ----- -ok - stabilize ---- ok log-level debug ----- -ok - status 1 ---- 1: StateReplicate match=11 next=12 @@ -43,11 +30,8 @@ status 1 2: StateProbe match=11 next=12 3: StateReplicate match=11 next=12 -tick-heartbeat 1 ----- -ok - # Heartbeat -> HeartbeatResp -> MsgApp -> MsgAppResp -> StateReplicate. +tick-heartbeat 1 stabilize ---- > 1 handling Ready diff --git a/testdata/prevote.txt b/testdata/prevote.txt index db763d35..c2eae026 100644 --- a/testdata/prevote.txt +++ b/testdata/prevote.txt @@ -6,30 +6,15 @@ # and is not enforced with PreVote alone. log-level none ----- -ok - add-nodes 3 voters=(1,2,3) index=10 prevote=true ----- -ok - campaign 1 ----- -ok - stabilize ---- ok -log-level debug ----- -ok - # Propose a command on 1 and replicate it to 2. +log-level debug propose 1 prop_1 ----- -ok - process-ready 1 ---- Ready MustSync=true: diff --git a/testdata/prevote_checkquorum.txt b/testdata/prevote_checkquorum.txt index 6db6662b..ee447883 100644 --- a/testdata/prevote_checkquorum.txt +++ b/testdata/prevote_checkquorum.txt @@ -4,26 +4,14 @@ # timeout interval, or if a quorum of voters are precandidates. log-level none ----- -ok - add-nodes 3 voters=(1,2,3) index=10 prevote=true checkquorum=true ----- -ok - campaign 1 ----- -ok - stabilize ---- ok -log-level debug ----- -ok - # 2 should fail to campaign, leaving 1's leadership alone. +log-level debug campaign 2 ---- INFO 2 is starting a new election at term 1 @@ -51,13 +39,7 @@ stabilize # If 2 hasn't heard from the leader in the past election timeout, it should # grant prevotes, allowing 3 to hold an election. set-randomized-election-timeout 2 timeout=5 ----- -ok - tick-election 2 ----- -ok - campaign 3 ---- INFO 3 is starting a new election at term 1 diff --git a/testdata/probe_and_replicate.txt b/testdata/probe_and_replicate.txt index d5829704..c6394b7e 100644 --- a/testdata/probe_and_replicate.txt +++ b/testdata/probe_and_replicate.txt @@ -28,61 +28,34 @@ # Set up the log configuration. This is mostly unintersting, but the order of # each leadership change and the nodes that are allowed to hear about them is # very important. Most readers of this test can skip this section. +# +# Start with seven nodes. Elect 1 as the leader. log-level none ----- -ok - -## Start with seven nodes. add-nodes 7 voters=(1,2,3,4,5,6,7) index=10 ----- -ok - -## Create term 1 entries. campaign 1 ----- -ok - stabilize ---- ok propose 1 prop_1_12 ----- -ok - propose 1 prop_1_13 ----- -ok - stabilize ---- ok ## Create term 2 entries. campaign 2 ----- -ok - stabilize 2 ---- ok stabilize 6 ----- -ok - stabilize 2 5 7 ---- ok propose 2 prop_2_15 ----- -ok - propose 2 prop_2_16 ----- -ok - stabilize 2 7 ---- ok @@ -93,37 +66,16 @@ ok ## Create term 3 entries. campaign 7 ----- -ok - stabilize 7 ----- -ok - stabilize 1 2 3 4 5 6 ---- ok stabilize 7 ----- -ok - propose 7 prop_3_18 ----- -ok - propose 7 prop_3_19 ----- -ok - propose 7 prop_3_20 ----- -ok - propose 7 prop_3_21 ----- -ok - stabilize 7 ---- ok @@ -134,29 +86,14 @@ ok ## Create term 4 entries. campaign 6 ----- -ok - stabilize 1 2 3 4 5 6 ---- ok propose 6 prop_4_15 ----- -ok - stabilize 1 2 4 5 6 ----- -ok - propose 6 prop_4_16 ----- -ok - propose 6 prop_4_17 ----- -ok - stabilize 6 ---- ok @@ -167,17 +104,8 @@ ok ## Create term 5 entries. campaign 5 ----- -ok - stabilize 1 2 4 5 ----- -ok - propose 5 prop_5_17 ----- -ok - stabilize 1 2 4 5 ---- ok @@ -188,33 +116,12 @@ ok ## Create term 6 entries. campaign 4 ----- -ok - stabilize 1 2 4 5 ----- -ok - propose 4 prop_6_19 ----- -ok - stabilize 1 2 4 ----- -ok - propose 4 prop_6_20 ----- -ok - stabilize 1 4 ----- -ok - propose 4 prop_6_21 ----- -ok - stabilize 4 ---- ok @@ -225,33 +132,12 @@ ok ## Create term 7 entries. campaign 5 ----- -ok - stabilize 5 ----- -ok - stabilize 1 3 6 7 ----- -ok - stabilize 5 ----- -ok - propose 5 prop_7_20 ----- -ok - propose 5 prop_7_21 ----- -ok - propose 5 prop_7_22 ----- -ok - stabilize 5 ---- ok @@ -263,9 +149,6 @@ ok # Show the Raft log from each node. log-level info ----- -ok - raft-log 1 ---- 1/11 EntryNormal "" diff --git a/testdata/replicate_pause.txt b/testdata/replicate_pause.txt index d9cee59f..05f57457 100644 --- a/testdata/replicate_pause.txt +++ b/testdata/replicate_pause.txt @@ -2,68 +2,38 @@ # in-flight state exceeds the configured limits. This is a regression test for # the issue fixed by https://github.com/etcd-io/etcd/pull/14633. -# Turn off output during the setup of the test. -log-level none ----- -ok - # Start with 3 nodes, with a limited in-flight capacity. +log-level none add-nodes 3 voters=(1,2,3) index=10 inflight=3 ----- -ok - campaign 1 ----- -ok - stabilize ---- ok -# Propose 3 entries. +# Propose 3 entries, store them, and send proposals. propose 1 prop_1_12 ----- -ok - propose 1 prop_1_13 ----- -ok - propose 1 prop_1_14 ----- -ok - -# Store entries and send proposals. process-ready 1 ---- ok -# Re-enable log messages. -log-level debug ----- -ok - # Expect that in-flight tracking to nodes 2 and 3 is saturated. +log-level debug status 1 +log-level none ---- 1: StateReplicate match=14 next=15 2: StateReplicate match=11 next=15 paused inflight=3[full] 3: StateReplicate match=11 next=15 paused inflight=3[full] -log-level none ----- -ok - # Commit entries between nodes 1 and 2. stabilize 1 2 ---- ok -log-level debug ----- -ok - # Expect that the entries are committed and stored on nodes 1 and 2. +log-level debug status 1 ---- 1: StateReplicate match=14 next=15 @@ -77,16 +47,9 @@ dropped: 1->3 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "prop_1 dropped: 1->3 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "prop_1_13"] dropped: 1->3 MsgApp Term:1 Log:1/13 Commit:11 Entries:[1/14 EntryNormal "prop_1_14"] - # Repeat committing 3 entries. propose 1 prop_1_15 ----- -ok - propose 1 prop_1_16 ----- -ok - propose 1 prop_1_17 ---- ok @@ -98,20 +61,14 @@ status 1 2: StateReplicate match=14 next=18 paused inflight=3[full] 3: StateReplicate match=11 next=15 paused inflight=3[full] -log-level none ----- -ok - # Commit entries between nodes 1 and 2 again. +log-level none stabilize 1 2 ---- ok -log-level debug ----- -ok - # Expect that the entries are committed and stored only on nodes 1 and 2. +log-level debug status 1 ---- 1: StateReplicate match=17 next=18 @@ -120,19 +77,14 @@ status 1 # Make a heartbeat roundtrip. tick-heartbeat 1 ----- -ok - stabilize 1 +stabilize 2 3 ---- > 1 handling Ready Ready MustSync=false: Messages: 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:17 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 - -stabilize 2 3 ----- > 2 receiving messages 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:17 > 3 receiving messages @@ -170,19 +122,10 @@ stabilize 3 Messages: 3->1 MsgAppResp Term:1 Log:1/14 Rejected (Hint: 11) +# Eventually all nodes catch up on the committed state. log-level none ----- -ok - stabilize ----- -ok - log-level debug ----- -ok - -# Eventually all nodes catch up on the committed state. status 1 ---- 1: StateReplicate match=17 next=18 diff --git a/testdata/single_node.txt b/testdata/single_node.txt index f6aceb71..7426d208 100644 --- a/testdata/single_node.txt +++ b/testdata/single_node.txt @@ -1,7 +1,4 @@ log-level info ----- -ok - add-nodes 1 voters=(1) index=3 ---- INFO 1 switched to configuration voters=(1) @@ -9,12 +6,10 @@ INFO 1 became follower at term 0 INFO newRaft 1 [peers: [1], term: 0, commit: 3, applied: 3, lastindex: 3, lastterm: 1] campaign 1 +stabilize ---- INFO 1 is starting a new election at term 0 INFO 1 became candidate at term 1 - -stabilize ----- > 1 handling Ready Ready MustSync=true: Lead:0 State:StateCandidate diff --git a/testdata/slow_follower_after_compaction.txt b/testdata/slow_follower_after_compaction.txt index 0d3d48c8..bac1d939 100644 --- a/testdata/slow_follower_after_compaction.txt +++ b/testdata/slow_follower_after_compaction.txt @@ -1,82 +1,41 @@ # This is a regression test for https://github.com/etcd-io/raft/pull/31. -# Turn off output during the setup of the test. -log-level none ----- -ok - # Start with 3 nodes, with a limited in-flight capacity. +log-level none add-nodes 3 voters=(1,2,3) index=10 inflight=2 ----- -ok - campaign 1 ----- -ok - stabilize ---- ok # Propose 3 entries. propose 1 prop_1_12 ----- -ok - propose 1 prop_1_13 ----- -ok - propose 1 prop_1_14 ----- -ok - stabilize ---- ok -# Re-enable log messages. -log-level debug ----- -ok - # All nodes up-to-date. +log-level debug status 1 ---- 1: StateReplicate match=14 next=15 2: StateReplicate match=14 next=15 3: StateReplicate match=14 next=15 +# Propose and commit entries on nodes 1 and 2. log-level none ----- -ok - propose 1 prop_1_15 ----- -ok - propose 1 prop_1_16 ----- -ok - propose 1 prop_1_17 ----- -ok - propose 1 prop_1_18 ----- -ok - -# Commit entries on nodes 1 and 2. stabilize 1 2 ---- ok -log-level debug ----- -ok - # Nodes 1 and 2 up-to-date, 3 is behind and MsgApp flow is throttled. +log-level debug status 1 ---- 1: StateReplicate match=18 next=19 @@ -98,22 +57,13 @@ compact 1 17 # which will reply with a rejection MsgApp because it sees a gap in the log. # Node 1 will reset the MsgApp flow and send a snapshot to catch node 3 up. tick-heartbeat 1 ----- -ok - log-level none ----- -ok - stabilize ---- ok -log-level debug ----- -ok - # All nodes caught up. +log-level debug status 1 ---- 1: StateReplicate match=18 next=19 diff --git a/testdata/snapshot_succeed_via_app_resp.txt b/testdata/snapshot_succeed_via_app_resp.txt index 80ed3646..9ec63a3e 100644 --- a/testdata/snapshot_succeed_via_app_resp.txt +++ b/testdata/snapshot_succeed_via_app_resp.txt @@ -6,21 +6,11 @@ # # See https://github.com/etcd-io/etcd/pull/10308 for additional background. -# Turn off output during the setup of the test. -log-level none ----- -ok - # Start with two nodes, but the config already has a third. +# Fully replicate everything, including the leader's empty index. +log-level none add-nodes 2 voters=(1,2,3) index=10 ----- -ok - campaign 1 ----- -ok - -# Fully replicate everything, including the leader's empty index. stabilize ---- ok @@ -34,11 +24,7 @@ deliver-msgs drop=(3) ---- ok -# Show the Raft log messages from now on. log-level debug ----- -ok - status 1 ---- 1: StateReplicate match=11 next=12 @@ -56,9 +42,6 @@ INFO newRaft 3 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastter # Time passes on the leader so that it will try the previously missing follower # again. tick-heartbeat 1 ----- -ok - process-ready 1 ---- Ready MustSync=false: diff --git a/testdata/snapshot_succeed_via_app_resp_behind.txt b/testdata/snapshot_succeed_via_app_resp_behind.txt index 09496982..6f33672b 100644 --- a/testdata/snapshot_succeed_via_app_resp_behind.txt +++ b/testdata/snapshot_succeed_via_app_resp_behind.txt @@ -1,16 +1,9 @@ # This is a variant of snapshot_succeed_via_app_resp in which the snapshot # that is being sent is behind the PendingSnapshot index tracked by the leader. -# Turn off output during the setup of the test. -log-level none ----- -ok - # Start with three nodes, but the third is disconnected from the log. +log-level none add-nodes 2 voters=(1,2,3) index=10 ----- -ok - add-nodes 1 voters=(1,2,3) index=5 ---- ok @@ -19,27 +12,15 @@ ok # and 2 to complete the leader election. We don't stabilize 3 after the # election, so that it does not receive and process any MsgApp yet. campaign 1 ----- -ok - process-ready 1 ----- -ok - stabilize 3 ----- -ok - stabilize 1 2 ---- ok -log-level debug ----- -ok - # We now have a leader at index 11 (it appended an empty entry when elected). 3 # is still at index 5, and has not received any MsgApp from the leader yet. +log-level debug raft-state ---- 1: StateLeader (Voter) Term:1 Lead:1 @@ -65,21 +46,12 @@ send-snapshot 1 3 # Propose and commit an additional entry, which makes the leader's # last index 12, beyond the snapshot it sent at index 11. log-level none ----- -ok - propose 1 "foo" ----- -ok - stabilize 1 2 ---- ok log-level debug ----- -ok - status 1 ---- 1: StateReplicate match=12 next=13 @@ -136,6 +108,7 @@ dropped: 1->3 MsgSnap Term:1 Log:0/0 Snapshot: Index:12 Term:1 ConfState:Voters: # 3 sends the affirmative MsgAppResp that resulted from applying the snapshot # at index 11. stabilize 3 +stabilize 1 ---- > 3 handling Ready Ready MustSync=false: @@ -143,9 +116,6 @@ stabilize 3 Snapshot Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false Messages: 3->1 MsgAppResp Term:1 Log:0/11 - -stabilize 1 ----- > 1 receiving messages 3->1 MsgAppResp Term:1 Log:0/11 DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 3 [StateSnapshot match=11 next=12 paused pendingSnap=12]