diff --git a/CHANGES.txt b/CHANGES.txt index 1cdbf799c155..153d38585f08 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 5.0.10 + * Clarify the unset, true and false behaviours of cassandra.reset_bootstrap_progress in the abort message and docs (CASSANDRA-21595) * Bound and chunk the BTI partition index preload (CASSANDRA-21632) * Restore volatile on CommitLogSegment.cdcState dropped in merge (CASSANDRA-21665) * Fix compressed scan read-ahead buffer Block leak and chunk-cache pollution by scans (CASSANDRA-21671) diff --git a/doc/modules/cassandra/pages/managing/operating/topo_changes.adoc b/doc/modules/cassandra/pages/managing/operating/topo_changes.adoc index 2bd85519b54c..6ecf3c5c5017 100644 --- a/doc/modules/cassandra/pages/managing/operating/topo_changes.adoc +++ b/doc/modules/cassandra/pages/managing/operating/topo_changes.adoc @@ -50,9 +50,41 @@ unavailable replica, set the JVM flag On 2.2+, if the bootstrap process fails, it's possible to resume bootstrap from the previous saved state by calling `nodetool bootstrap resume`. If for some reason the bootstrap hangs or -stalls, it may also be resumed by simply restarting the node. In order -to cleanup bootstrap state and start fresh, you may set the JVM startup -flag `-Dcassandra.reset_bootstrap_progress=true`. +stalls, it may also be resumed by simply restarting the node. + +==== The `cassandra.reset_bootstrap_progress` JVM property + +When a bootstrap finds ranges that a previous, partially successful +bootstrap already marked as available, the behaviour depends on the +`cassandra.reset_bootstrap_progress` JVM startup property. Leaving it +unset is *not* the same as setting it to `true` or to `false`. + +[cols=",",options="header",] +|=== +|Value |Behaviour when previous bootstrap progress is found + +|unset (default) | If ranges from a previous bootstrap are found, +the bootstrap usually aborts with an error, so that no decision +about the existing data is made on your behalf. If none are found, +the bootstrap proceeds normally. + +|`true` |Discards the recorded progress and streams all ranges again. +The files already streamed are *not* deleted, so remove them before +retrying or you may resurrect or duplicate data. See +https://issues.apache.org/jira/browse/CASSANDRA-17679[CASSANDRA-17679]. + +|`false` |Keeps the recorded progress and streams only the ranges not +already marked as available. Potentially dangerous, because the data +streamed by the previous attempt may be missing or incomplete. Prefer +`true` after cleaning up the old files. See +https://issues.apache.org/jira/browse/CASSANDRA-17679[CASSANDRA-17679]. +|=== + +In some situations, a bootstrap with the property unset doesn't abort. +Instead, it continues and reuses the ranges that a previous attempt +already streamed, without logging that it skipped them. To avoid this, +we recommend setting the property to `true` after removing the files +from the previous attempt. On lower versions, when the bootstrap proces fails it is recommended to wipe the node (remove all the data), and restart the bootstrap process diff --git a/src/java/org/apache/cassandra/dht/RangeStreamer.java b/src/java/org/apache/cassandra/dht/RangeStreamer.java index a95a7c300e3e..501d228f08a8 100644 --- a/src/java/org/apache/cassandra/dht/RangeStreamer.java +++ b/src/java/org/apache/cassandra/dht/RangeStreamer.java @@ -735,15 +735,18 @@ public StreamResultFuture fetchAsync() if (remaining.size() < available.full.size() + available.trans.size()) { - // If the operator hasn't specified what to do when we discover a previous partially successful bootstrap, - // we error out and tell them to manually reconcile it. See CASSANDRA-17679. + // If the operator hasn't specified what to do when we discover a previous partially successful bootstrap + // (some of the ranges we are about to fetch are already marked as available), we error out and spell out + // the true/false choice. See CASSANDRA-17679. if (!RESET_BOOTSTRAP_PROGRESS.isPresent()) { List skipped = fetchReplicas.stream().filter(isAvailable).collect(Collectors.toList()); - String msg = String.format("Discovered existing bootstrap data and %s " + - "is not configured; aborting bootstrap. Please clean up local files manually " + - "and try again or set cassandra.reset_bootstrap_progress=true to ignore. " + - "Found: %s. Fully available: %s. Transiently available: %s", + String msg = String.format("Discovered existing bootstrap data (ranges already marked as available by a previous " + + "bootstrap attempt) and %1$s is unset; aborting bootstrap. To start a new bootstrap and " + + "re-stream all ranges, delete the data files from the previous attempt and set it to 'true'. " + + "To stream only the missing ranges, set it to 'false' (potentially dangerous, as the previous " + + "data may be incomplete). Set it on restart with -D%1$s=. See CASSANDRA-17679. " + + "Found: %2$s. Fully available: %3$s. Transiently available: %4$s", RESET_BOOTSTRAP_PROGRESS.getKey(), skipped, available.full, available.trans); logger.error(msg); throw new IllegalStateException(msg);