diff --git a/gradle/testing/randomization/policies/solr-tests.policy b/gradle/testing/randomization/policies/solr-tests.policy index 12c44a8e5b4..a17b015b900 100644 --- a/gradle/testing/randomization/policies/solr-tests.policy +++ b/gradle/testing/randomization/policies/solr-tests.policy @@ -161,6 +161,13 @@ grant { // used by solr to create sandboxes (e.g. script execution) permission java.security.SecurityPermission "createAccessControlContext"; + + // used by gcs-repository + permission java.net.SocketPermission "metadata.google.internal:80", "connect,resolve"; + permission java.net.SocketPermission "www.googleapis.com:443", "connect,resolve"; + permission java.net.SocketPermission "oauth2.googleapis.com:443", "connect,resolve"; + // used on jenkins for loading credentials + permission java.io.FilePermission "${user.home}${/}.config${/}gcloud${/}-", "read"; }; // additional permissions based on system properties set by /bin/solr diff --git a/settings.gradle b/settings.gradle index be2c09cc235..eab92ef92ae 100644 --- a/settings.gradle +++ b/settings.gradle @@ -66,6 +66,7 @@ include "solr:contrib:langid" include "solr:contrib:jaegertracer-configurator" include "solr:contrib:prometheus-exporter" include "solr:contrib:ltr" +include "solr:contrib:gcs-repository" include "solr:webapp" include "solr:test-framework" include "solr:solr-ref-guide" diff --git a/solr/contrib/gcs-repository/README.md b/solr/contrib/gcs-repository/README.md new file mode 100644 index 00000000000..ee966318977 --- /dev/null +++ b/solr/contrib/gcs-repository/README.md @@ -0,0 +1,4 @@ +Apache Solr - GCS Respository +============================= + +The GCS repository is a backup repository implementation that uses Google Cloud Storage (GCS). \ No newline at end of file diff --git a/solr/contrib/gcs-repository/build.gradle b/solr/contrib/gcs-repository/build.gradle new file mode 100644 index 00000000000..26ac132c7f8 --- /dev/null +++ b/solr/contrib/gcs-repository/build.gradle @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +apply plugin: 'java-library' + +description = 'GCS Backup Repository' + +dependencies { + api project(':solr:core') + + implementation ('com.google.api:api-common') { transitive = false } + implementation ('com.google.api:gax') { transitive = false } + implementation ('com.google.api:gax-httpjson') { transitive = false } + implementation ('com.google.cloud:google-cloud-core') { transitive = false } + implementation ('com.google.cloud:google-cloud-core-http') { transitive = false } + implementation ('com.google.cloud:google-cloud-storage') { transitive = false } + implementation ('com.google.auth:google-auth-library-oauth2-http') { transitive = false } + implementation ('com.google.auth:google-auth-library-credentials') { transitive = false } + implementation ('org.threeten:threetenbp') { transitive = false } + + runtimeOnly ('com.google.apis:google-api-services-storage') { transitive = false } + runtimeOnly ('com.google.api-client:google-api-client') { transitive = false } + runtimeOnly ('com.google.api.grpc:proto-google-common-protos') { transitive = false } + runtimeOnly ('com.google.api.grpc:proto-google-iam-v1') { transitive = false } + runtimeOnly ('com.google.code.findbugs:jsr305') { transitive = false } + runtimeOnly ('com.google.code.gson:gson') { transitive = false } + runtimeOnly ('com.google.http-client:google-http-client') { transitive = false } + runtimeOnly ('com.google.http-client:google-http-client-appengine') { transitive = false } + runtimeOnly ('com.google.http-client:google-http-client-jackson2') { transitive = false } + runtimeOnly ('com.google.j2objc:j2objc-annotations') { transitive = false } + runtimeOnly ('com.google.oauth-client:google-oauth-client') { transitive = false } + runtimeOnly ('com.google.protobuf:protobuf-java') { transitive = false } + runtimeOnly ('com.google.protobuf:protobuf-java-util') { transitive = false } + runtimeOnly ('io.grpc:grpc-context') { transitive = false } + runtimeOnly ('io.opencensus:opencensus-api') { transitive = false } + runtimeOnly ('io.opencensus:opencensus-contrib-http-util') { transitive = false } + + testImplementation project(':solr:test-framework') +} diff --git a/solr/contrib/gcs-repository/src/java/org/apache/solr/gcs/GCSBackupRepository.java b/solr/contrib/gcs-repository/src/java/org/apache/solr/gcs/GCSBackupRepository.java new file mode 100644 index 00000000000..089ea2787a3 --- /dev/null +++ b/solr/contrib/gcs-repository/src/java/org/apache/solr/gcs/GCSBackupRepository.java @@ -0,0 +1,472 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.gcs; + +import com.google.api.gax.retrying.RetrySettings; +import com.google.auth.oauth2.GoogleCredentials; +import com.google.cloud.ReadChannel; +import com.google.cloud.WriteChannel; +import com.google.cloud.storage.*; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.store.*; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.DirectoryFactory; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.threeten.bp.Duration; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.lang.invoke.MethodHandles; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.NoSuchFileException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import static java.net.HttpURLConnection.HTTP_PRECON_FAILED; + +public class GCSBackupRepository implements BackupRepository { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final int LARGE_BLOB_THRESHOLD_BYTE_SIZE = 5 * 1024 * 1024; + private static final int BUFFER_SIZE = 16 * 1024 * 1024; + private static Storage storage; + + @SuppressWarnings("rawtypes") + private NamedList config = null; + private String bucketName = "backup-managed-solr"; + + static synchronized Storage initStorage(String credentialPath) { + if (storage != null) + return storage; + + try { + if (credentialPath == null) { + credentialPath = System.getenv("GCS_CREDENTIAL"); + if (credentialPath == null) { + credentialPath = System.getenv("BACKUP_CREDENTIAL"); + } + } + StorageOptions.Builder builder = StorageOptions.newBuilder(); + if (credentialPath != null) { + log.info("Creating GCS client using credential at {}", credentialPath); + GoogleCredentials credential = GoogleCredentials.fromStream(new FileInputStream(credentialPath)); + builder.setCredentials(credential); + } + storage = builder + .setTransportOptions(StorageOptions.getDefaultHttpTransportOptions().toBuilder() + .setConnectTimeout(20000) + .setReadTimeout(20000) + .build()) + .setRetrySettings(RetrySettings.newBuilder().setMaxAttempts(10) + //http requests + .setInitialRetryDelay(Duration.ofSeconds(1)) + .setMaxRetryDelay(Duration.ofSeconds(30)) + .setRetryDelayMultiplier(1.0) + //rpc requests + .setInitialRpcTimeout(Duration.ofSeconds(10)) + .setMaxRpcTimeout(Duration.ofSeconds(30)) + .setRpcTimeoutMultiplier(1.0) + //total retry timeout + .setTotalTimeout(Duration.ofSeconds(300)) + .build()) + .build().getService(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + return storage; + } + + @Override + public void init(@SuppressWarnings("rawtypes") NamedList args) { + this.config = args; + initStorage((String)args.get("credential")); + + if (args.get("bucket") != null) { + this.bucketName = args.get("bucket").toString(); + } else { + this.bucketName = System.getenv("GCS_BUCKET"); + if (this.bucketName == null) { + this.bucketName = System.getenv().getOrDefault("BACKUP_BUCKET","backup-managed-solr"); + } + } + } + + @Override + @SuppressWarnings("unchecked") + public T getConfigProperty(String name) { + return (T) this.config.get(name); + } + + @Override + public URI createURI(String location) { + Objects.requireNonNull(location); + + URI result; + try { + result = new URI(location); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Error on creating URI", e); + } + + return result; + } + + @Override + public URI resolve(URI baseUri, String... pathComponents) { + StringBuilder builder = new StringBuilder(baseUri.toString()); + for (String path : pathComponents) { + if (path != null && !path.isEmpty()) { + if (builder.charAt(builder.length()-1) != '/') { + builder.append('/'); + } + builder.append(path); + } + } + + return URI.create(builder.toString()); + } + + @Override + public boolean exists(URI path) throws IOException { + if (path.toString().equals(getConfigProperty("location"))) { + return true; + } + + if (path.toString().endsWith("/")) { + return storage.get(bucketName, path.toString(), Storage.BlobGetOption.fields()) != null; + } else { + List req = Arrays.asList( + BlobId.of(bucketName, path.toString()), + BlobId.of(bucketName, path.toString() + "/")); + List rs = storage.get(req); + return rs.get(0) != null || rs.get(1) != null; + } + + } + + @Override + public PathType getPathType(URI path) throws IOException { + if (path.toString().endsWith("/")) + return PathType.DIRECTORY; + + Blob blob = storage.get(bucketName, path.toString()+"/", Storage.BlobGetOption.fields()); + if (blob != null) + return PathType.DIRECTORY; + + return PathType.FILE; + } + + private String toBlobName(URI path) { + return path.toString(); + } + + @Override + public String[] listAll(URI path) throws IOException { + String blobName = path.toString(); + if (!blobName.endsWith("/")) + blobName += "/"; + + final String pathStr = blobName; + final LinkedList result = new LinkedList<>(); + storage.get(bucketName).list( + Storage.BlobListOption.currentDirectory(), + Storage.BlobListOption.prefix(pathStr), + Storage.BlobListOption.fields() + ).iterateAll().forEach( + blob -> { + assert blob.getName().startsWith(pathStr); + final String suffixName = blob.getName().substring(pathStr.length()); + if (!suffixName.isEmpty()) { + result.add(suffixName); + } + }); + + return result.toArray(new String[0]); + } + + @Override + public IndexInput openInput(URI dirPath, String fileName, IOContext ctx) throws IOException { + return openInput(dirPath, fileName, ctx, 2 * 1024 * 1024); + } + + private IndexInput openInput(URI dirPath, String fileName, IOContext ctx, int bufferSize) { + String blobName = dirPath.toString(); + if (!blobName.endsWith("/")) { + blobName += "/"; + } + blobName += fileName; + + final BlobId blobId = BlobId.of(bucketName, blobName); + final Blob blob = storage.get(blobId, Storage.BlobGetOption.fields(Storage.BlobField.SIZE)); + final ReadChannel readChannel = blob.reader(); + readChannel.setChunkSize(bufferSize); + + return new BufferedIndexInput(blobName, bufferSize) { + + @Override + public long length() { + return blob.getSize(); + } + + @Override + protected void readInternal(ByteBuffer b) throws IOException { + readChannel.read(b); + } + + @Override + protected void seekInternal(long pos) throws IOException { + readChannel.seek(pos); + } + + @Override + public void close() throws IOException { + readChannel.close(); + } + }; + } + + @Override + public OutputStream createOutput(URI path) throws IOException { + final BlobInfo blobInfo = BlobInfo.newBuilder(bucketName, toBlobName(path)).build(); + final Storage.BlobWriteOption[] writeOptions = new Storage.BlobWriteOption[0]; + final WriteChannel writeChannel = storage.writer(blobInfo, writeOptions); + + return Channels.newOutputStream(new WritableByteChannel() { + @Override + public int write(ByteBuffer src) throws IOException { + return writeChannel.write(src); + } + + @Override + public boolean isOpen() { + return writeChannel.isOpen(); + } + + @Override + public void close() throws IOException { + writeChannel.close(); + } + }); + } + + @Override + public void createDirectory(URI path) throws IOException { + String name = path.toString(); + if (!name.endsWith("/")) + name += "/"; + storage.create(BlobInfo.newBuilder(bucketName, name).build()) ; + } + + @Override + public void deleteDirectory(URI path) throws IOException { + List blobIds = allBlobsAtDir(path); + if (!blobIds.isEmpty()) { + storage.delete(blobIds); + } else { + log.info("Path:{} doesn't have any blobs", path); + } + } + + private List allBlobsAtDir(URI path) throws IOException { + String blobName = path.toString(); + if (!blobName.endsWith("/")) + blobName += "/"; + + final List result = new ArrayList<>(); + final String pathStr = blobName; + storage.get(bucketName).list( + Storage.BlobListOption.prefix(pathStr), + Storage.BlobListOption.fields() + ).iterateAll().forEach( + blob -> result.add(blob.getBlobId()) + ); + + return result; + + } + + @Override + public void delete(URI path, Collection files, boolean ignoreNoSuchFileException) throws IOException { + if (files.isEmpty()) { + return; + } + String prefix; + if (path.toString().endsWith("/")) { + prefix = path.toString(); + } else { + prefix = path.toString() + "/"; + } + List blobDeletes = files.stream() + .map(file -> BlobId.of(bucketName, prefix + file)) + .collect(Collectors.toList()); + List result = storage.delete(blobDeletes); + if (!ignoreNoSuchFileException) { + int failedDelete = result.indexOf(Boolean.FALSE); + if (failedDelete != -1) { + throw new NoSuchFileException("File " + blobDeletes.get(failedDelete).getName() + " was not found"); + } + } + } + + @Override + public void copyIndexFileFrom(Directory sourceDir, String sourceFileName, URI destDir, String destFileName) throws IOException { + String blobName = destDir.toString(); + if (!blobName.endsWith("/")) + blobName += "/"; + blobName += destFileName; + final BlobInfo blobInfo = BlobInfo.newBuilder(bucketName, blobName).build(); + try (ChecksumIndexInput input = sourceDir.openChecksumInput(sourceFileName, DirectoryFactory.IOCONTEXT_NO_CACHE)) { + if (input.length() <= CodecUtil.footerLength()) { + throw new CorruptIndexException("file is too small:" + input.length(), input); + } + if (input.length() > LARGE_BLOB_THRESHOLD_BYTE_SIZE) { + writeBlobResumable(blobInfo, input); + } else { + writeBlobMultipart(blobInfo, input, (int) input.length()); + } + } + } + + @Override + public void copyIndexFileTo(URI sourceRepo, String sourceFileName, Directory dest, String destFileName) throws IOException { + String blobName = sourceRepo.toString(); + if (!blobName.endsWith("/")) + blobName += "/"; + blobName += sourceFileName; + final BlobId blobId = BlobId.of(bucketName, blobName); + try (final ReadChannel readChannel = storage.reader(blobId); + IndexOutput output = dest.createOutput(destFileName, DirectoryFactory.IOCONTEXT_NO_CACHE)) { + ByteBuffer buffer = ByteBuffer.allocate(1024 * 1024 * 8); + while (readChannel.read(buffer) > 0) { + buffer.flip(); + byte[] arr = buffer.array(); + output.writeBytes(arr, buffer.position(), buffer.limit() - buffer.position()); + buffer.clear(); + } + } + + } + + + @Override + public void close() throws IOException { + + } + + private void writeBlobMultipart(BlobInfo blobInfo, ChecksumIndexInput indexInput, int blobSize) + throws IOException { + byte[] bytes = new byte[blobSize]; + indexInput.readBytes(bytes, 0, blobSize - CodecUtil.footerLength()); + long checksum = CodecUtil.checkFooter(indexInput); + ByteBuffer footerBuffer = ByteBuffer.wrap(bytes, blobSize - CodecUtil.footerLength(), CodecUtil.footerLength()); + writeFooter(checksum, footerBuffer); + try { + storage.create(blobInfo, bytes, Storage.BlobTargetOption.doesNotExist()); + } catch (final StorageException se) { + if (se.getCode() == HTTP_PRECON_FAILED) { + throw new FileAlreadyExistsException(blobInfo.getBlobId().getName(), null, se.getMessage()); + } + throw se; + } + } + + private void writeBlobResumable(BlobInfo blobInfo, ChecksumIndexInput indexInput) throws IOException { + try { + final Storage.BlobWriteOption[] writeOptions = new Storage.BlobWriteOption[0]; + final WriteChannel writeChannel = storage.writer(blobInfo, writeOptions); + + ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE); + writeChannel.setChunkSize(BUFFER_SIZE); + + long remain = indexInput.length() - CodecUtil.footerLength(); + while (remain > 0) { + // reading + int byteReads = (int) Math.min(buffer.capacity(), remain); + indexInput.readBytes(buffer.array(), 0, byteReads); + buffer.position(byteReads); + buffer.flip(); + + // writing + writeChannel.write(buffer); + buffer.clear(); + remain -= byteReads; + } + long checksum = CodecUtil.checkFooter(indexInput); + ByteBuffer bytes = getFooter(checksum); + writeChannel.write(bytes); + writeChannel.close(); + } catch (final StorageException se) { + if (se.getCode() == HTTP_PRECON_FAILED) { + throw new FileAlreadyExistsException(blobInfo.getBlobId().getName(), null, se.getMessage()); + } + throw se; + } + } + + private ByteBuffer getFooter(long checksum) throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(CodecUtil.footerLength()); + writeFooter(checksum, buffer); + return buffer; + } + + private void writeFooter(long checksum, ByteBuffer buffer) throws IOException { + IndexOutput out = new IndexOutput("", "") { + + @Override + public void writeByte(byte b) throws IOException { + buffer.put(b); + } + + @Override + public void writeBytes(byte[] b, int offset, int length) throws IOException { + buffer.put(b, offset, length); + } + + @Override + public void close() throws IOException { + + } + + @Override + public long getFilePointer() { + return 0; + } + + @Override + public long getChecksum() throws IOException { + return checksum; + } + }; + CodecUtil.writeFooter(out); + buffer.flip(); + } + +} diff --git a/solr/contrib/gcs-repository/src/java/org/apache/solr/gcs/package-info.java b/solr/contrib/gcs-repository/src/java/org/apache/solr/gcs/package-info.java new file mode 100644 index 00000000000..e209187846b --- /dev/null +++ b/solr/contrib/gcs-repository/src/java/org/apache/solr/gcs/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * GCS Backup Repository for Solr + */ +package org.apache.solr.gcs; \ No newline at end of file diff --git a/solr/contrib/gcs-repository/src/test-files/conf/schema.xml b/solr/contrib/gcs-repository/src/test-files/conf/schema.xml new file mode 100644 index 00000000000..4124feab0c3 --- /dev/null +++ b/solr/contrib/gcs-repository/src/test-files/conf/schema.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + id + diff --git a/solr/contrib/gcs-repository/src/test-files/conf/solrconfig.xml b/solr/contrib/gcs-repository/src/test-files/conf/solrconfig.xml new file mode 100644 index 00000000000..853ba656241 --- /dev/null +++ b/solr/contrib/gcs-repository/src/test-files/conf/solrconfig.xml @@ -0,0 +1,51 @@ + + + + + + + + + ${solr.data.dir:} + + + + + ${tests.luceneMatchVersion:LATEST} + + + + ${solr.commitwithin.softcommit:true} + + + + + + + explicit + true + text + + + + + +: + + diff --git a/solr/contrib/gcs-repository/src/test-files/log4j2.xml b/solr/contrib/gcs-repository/src/test-files/log4j2.xml new file mode 100644 index 00000000000..46ad20c18ee --- /dev/null +++ b/solr/contrib/gcs-repository/src/test-files/log4j2.xml @@ -0,0 +1,69 @@ + + + + + + + + + %maxLen{%-4r %-5p (%t) [%X{node_name} %X{collection} %X{shard} %X{replica} %X{core} %X{trace_id}] %c{1.} %m%notEmpty{ + =>%ex{short}}}{10240}%n + + + + + + + + + + + + + + + + + + + diff --git a/solr/contrib/gcs-repository/src/test/org/apache/solr/gcs/GCSBackupRepositoryTest.java b/solr/contrib/gcs-repository/src/test/org/apache/solr/gcs/GCSBackupRepositoryTest.java new file mode 100644 index 00000000000..b6f555e7cdb --- /dev/null +++ b/solr/contrib/gcs-repository/src/test/org/apache/solr/gcs/GCSBackupRepositoryTest.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.gcs; + +import java.net.URI; +import java.net.URISyntaxException; + +import org.apache.solr.cloud.api.collections.AbstractBackupRepositoryTest; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.backup.repository.BackupRepository; + +public class GCSBackupRepositoryTest extends AbstractBackupRepositoryTest { + @Override + @SuppressWarnings("rawtypes") + protected BackupRepository getRepository() { + GCSBackupRepository repository = new GCSBackupRepository(); + repository.init(new NamedList()); + return repository; + } + + @Override + protected URI getBaseUri() throws URISyntaxException { + return new URI("/tmp"); + } +} diff --git a/solr/contrib/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java b/solr/contrib/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java new file mode 100644 index 00000000000..3109d24bd84 --- /dev/null +++ b/solr/contrib/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.gcs; + +import java.lang.invoke.MethodHandles; +import java.util.List; +import java.util.stream.Collectors; + +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.common.collect.Lists; +import org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest; +import org.junit.BeforeClass; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GCSIncrementalBackupTest extends AbstractIncrementalBackupTest { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final String SOLR_XML = "\n" + + "\n" + + " ${shareSchema:false}\n" + + " ${configSetBaseDir:configsets}\n" + + " ${coreRootDirectory:.}\n" + + "\n" + + " \n" + + " ${urlScheme:}\n" + + " ${socketTimeout:90000}\n" + + " ${connTimeout:15000}\n" + + " \n" + + "\n" + + " \n" + + " 127.0.0.1\n" + + " ${hostPort:8983}\n" + + " ${hostContext:solr}\n" + + " ${solr.zkclienttimeout:30000}\n" + + " ${genericCoreNodeNames:true}\n" + + " 10000\n" + + " ${distribUpdateConnTimeout:45000}\n" + + " ${distribUpdateSoTimeout:340000}\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " localfs\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "\n"; + + private static String backupLocation; + + @BeforeClass + public static void setupClass() throws Exception { + Storage storage = GCSBackupRepository.initStorage(null); + List existingBlobs = Lists.newArrayList(storage.list("backup-managed-solr").iterateAll()) + .stream() + .map(BlobInfo::getBlobId) + .collect(Collectors.toList()); + if (!existingBlobs.isEmpty()) { + storage.delete(existingBlobs); + } + + configureCluster(NUM_SHARDS)// nodes + .addConfig("conf1", getFile("conf/solrconfig.xml").getParentFile().toPath()) + .withSolrXml(SOLR_XML) + .configure(); + } + + @Override + public String getCollectionNamePrefix() { + return "backuprestore"; + } + + @Override + public String getBackupLocation() { + return "/backup1"; + } + +} diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java index 61be096277a..275a9b5fbcd 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java @@ -22,15 +22,13 @@ import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; +import java.io.IOException; import java.lang.invoke.MethodHandles; import java.net.URI; -import java.time.Instant; import java.util.Collection; import java.util.Collections; import java.util.Optional; -import java.util.Properties; -import org.apache.lucene.util.Version; import org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.ShardRequestTracker; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; @@ -47,11 +45,13 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.backup.BackupManager; +import org.apache.solr.core.backup.BackupProperties; import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.core.snapshots.CollectionSnapshotMetaData; import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.CoreSnapshotMetaData; import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.SnapshotStatus; import org.apache.solr.core.snapshots.SolrSnapshotManager; +import org.apache.solr.handler.IncrementalBackupPaths; import org.apache.solr.handler.component.ShardHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,7 +66,9 @@ public BackupCmd(OverseerCollectionMessageHandler ocmh) { } @Override - public void call(ClusterState state, ZkNodeProps message, @SuppressWarnings({"rawtypes"})NamedList results) throws Exception { + @SuppressWarnings({"unchecked"}) + public void call(ClusterState state, ZkNodeProps message, @SuppressWarnings({"rawtypes"}) NamedList results) throws Exception { + String extCollectionName = message.getStr(COLLECTION_PROP); boolean followAliases = message.getBool(FOLLOW_ALIASES, false); String collectionName; @@ -77,64 +79,81 @@ public void call(ClusterState state, ZkNodeProps message, @SuppressWarnings({"ra } String backupName = message.getStr(NAME); String repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY); + boolean incremental = message.getBool(CoreAdminParams.BACKUP_INCREMENTAL, false); + String configName = ocmh.zkStateReader.readConfigName(collectionName); - Instant startTime = Instant.now(); + BackupProperties backupProperties = BackupProperties.create(backupName, collectionName, + extCollectionName, configName); CoreContainer cc = ocmh.overseer.getCoreContainer(); - BackupRepository repository = cc.newBackupRepository(repo); - BackupManager backupMgr = new BackupManager(repository, ocmh.zkStateReader); - - // Backup location - URI location = repository.createURI(message.getStr(CoreAdminParams.BACKUP_LOCATION)); - URI backupPath = repository.resolve(location, backupName); - - //Validating if the directory already exists. - if (repository.exists(backupPath)) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The backup directory already exists: " + backupPath); - } - - // Create a directory to store backup details. - repository.createDirectory(backupPath); - - String strategy = message.getStr(CollectionAdminParams.INDEX_BACKUP_STRATEGY, CollectionAdminParams.COPY_FILES_STRATEGY); - switch (strategy) { - case CollectionAdminParams.COPY_FILES_STRATEGY: { - copyIndexFiles(backupPath, collectionName, message, results); - break; + try (BackupRepository repository = cc.newBackupRepository(repo)) { + + // Backup location + URI location = repository.createURI(message.getStr(CoreAdminParams.BACKUP_LOCATION)); + URI backupPath = repository.resolve(location, backupName); + + //Validating if the directory already exists. + if (!repository.exists(backupPath)) { + // Create a directory to store backup details. + repository.createDirectory(backupPath); + } else if(!incremental) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The backup directory already exists: " + backupPath); } - case CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY: { - break; + + BackupManager backupMgr; + if (incremental) { + IncrementalBackupPaths incBackupFiles = new IncrementalBackupPaths(repository, backupPath); + incBackupFiles.createFolders(); + backupMgr = BackupManager.forIncrementalBackup(repository, ocmh.zkStateReader, backupPath); + } else { + backupMgr = BackupManager.forBackup(repository, ocmh.zkStateReader, location, backupName); } - } - log.info("Starting to backup ZK data for backupName={}", backupName); + String strategy = message.getStr(CollectionAdminParams.INDEX_BACKUP_STRATEGY, CollectionAdminParams.COPY_FILES_STRATEGY); + switch (strategy) { + case CollectionAdminParams.COPY_FILES_STRATEGY: { + if (incremental) { + try { + incrementalCopyIndexFiles(backupPath, collectionName, message, results, backupProperties, backupMgr); + } catch (SolrException e) { + log.error("Error happened during incremental backup for collection:{}", collectionName, e); + ocmh.cleanBackup(repository, backupPath, backupMgr.getBackupId()); + throw e; + } + } else { + copyIndexFiles(backupPath, collectionName, message, results); + } + break; + } + case CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY: { + break; + } + } - //Download the configs - String configName = ocmh.zkStateReader.readConfigName(collectionName); - backupMgr.downloadConfigDir(location, backupName, configName); + log.info("Starting to backup ZK data for backupName={}", backupName); - //Save the collection's state (coming from the collection's state.json) - //We extract the state and back it up as a separate json - DocCollection collectionState = ocmh.zkStateReader.getClusterState().getCollection(collectionName); - backupMgr.writeCollectionState(location, backupName, collectionName, collectionState); + //Download the configs + backupMgr.downloadConfigDir(configName); - Properties properties = new Properties(); + //Save the collection's state. Can be part of the monolithic clusterstate.json or a individual state.json + //Since we don't want to distinguish we extract the state and back it up as a separate json + DocCollection collectionState = ocmh.zkStateReader.getClusterState().getCollection(collectionName); + backupMgr.writeCollectionState(collectionName, collectionState); + backupMgr.downloadCollectionProperties(collectionName); - properties.put(BackupManager.BACKUP_NAME_PROP, backupName); - properties.put(BackupManager.COLLECTION_NAME_PROP, collectionName); - properties.put(BackupManager.COLLECTION_ALIAS_PROP, extCollectionName); - properties.put(CollectionAdminParams.COLL_CONF, configName); - properties.put(BackupManager.START_TIME_PROP, startTime.toString()); - properties.put(BackupManager.INDEX_VERSION_PROP, Version.LATEST.toString()); - //TODO: Add MD5 of the configset. If during restore the same name configset exists then we can compare checksums to see if they are the same. - //if they are not the same then we can throw an error or have an 'overwriteConfig' flag - //TODO save numDocs for the shardLeader. We can use it to sanity check the restore. + //TODO: Add MD5 of the configset. If during restore the same name configset exists then we can compare checksums to see if they are the same. + //if they are not the same then we can throw an error or have an 'overwriteConfig' flag + //TODO save numDocs for the shardLeader. We can use it to sanity check the restore. - backupMgr.writeBackupProperties(location, backupName, properties); + backupMgr.writeBackupProperties(backupProperties); - backupMgr.downloadCollectionProperties(location, backupName, collectionName); + log.info("Completed backing up ZK data for backupName={}", backupName); - log.info("Completed backing up ZK data for backupName={}", backupName); + int maxNumBackup = message.getInt(CoreAdminParams.MAX_NUM_BACKUP, -1); + if (incremental && maxNumBackup != -1) { + ocmh.deleteBackup(repository, backupPath, maxNumBackup, results); + } + } } private Replica selectReplicaWithSnapshot(CollectionSnapshotMetaData snapshotMeta, Slice slice) { @@ -142,9 +161,9 @@ private Replica selectReplicaWithSnapshot(CollectionSnapshotMetaData snapshotMet // If that is not possible, we choose any other replica for the given shard. Collection snapshots = snapshotMeta.getReplicaSnapshotsForShard(slice.getName()); - Optional leaderCore = snapshots.stream().filter(x -> x.isLeader()).findFirst(); + Optional leaderCore = snapshots.stream().filter(CoreSnapshotMetaData::isLeader).findFirst(); if (leaderCore.isPresent()) { - if (log.isInfoEnabled()) { + if (log.isInfoEnabled()) { log.info("Replica {} was the leader when snapshot {} was created.", leaderCore.get().getCoreName(), snapshotMeta.getName()); } Replica r = slice.getReplica(leaderCore.get().getCoreName()); @@ -154,19 +173,102 @@ private Replica selectReplicaWithSnapshot(CollectionSnapshotMetaData snapshotMet } Optional r = slice.getReplicas().stream() - .filter(x -> x.getState() != State.DOWN && snapshotMeta.isSnapshotExists(slice.getName(), x)) - .findFirst(); + .filter(x -> x.getState() != State.DOWN && snapshotMeta.isSnapshotExists(slice.getName(), x)) + .findFirst(); if (!r.isPresent()) { throw new SolrException(ErrorCode.SERVER_ERROR, - "Unable to find any live replica with a snapshot named " + snapshotMeta.getName() + " for shard " + slice.getName()); + "Unable to find any live replica with a snapshot named " + snapshotMeta.getName() + " for shard " + slice.getName()); } return r.get(); } + private void incrementalCopyIndexFiles(URI backupPath, String collectionName, ZkNodeProps request, + NamedList results, BackupProperties backupProperties, + BackupManager backupManager) throws IOException { + String backupName = request.getStr(NAME); + String asyncId = request.getStr(ASYNC); + String repoName = request.getStr(CoreAdminParams.BACKUP_REPOSITORY); + ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler(); + + log.info("Starting backup of collection={} with backupName={} at location={}", collectionName, backupName, + backupPath); + + Optional previousProps = backupManager.tryReadBackupProperties(); + final ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId); + + Collection slices = ocmh.zkStateReader.getClusterState().getCollection(collectionName).getActiveSlices(); + for (Slice slice : slices) { + // Note - Actually this can return a null value when there is no leader for this shard. + Replica replica = slice.getLeader(); + if (replica == null) { + throw new SolrException(ErrorCode.SERVER_ERROR, "No 'leader' replica available for shard " + slice.getName() + " of collection " + collectionName); + } + String coreName = replica.getStr(CORE_NAME_PROP); + + ModifiableSolrParams params = coreBackupParams(backupPath, repoName, slice, coreName); + params.set(CoreAdminParams.BACKUP_INCREMENTAL, true); + previousProps.flatMap(bp -> bp.getShardBackupIdFor(slice.getName())) + .ifPresent(prevBackupPoint -> params.set(CoreAdminParams.PREV_SHARD_BACKUP_ID, prevBackupPoint)); + + String shardBackupId = backupProperties.putAndGetShardBackupIdFor(slice.getName(), + backupManager.getBackupId().getId()); + params.set(CoreAdminParams.SHARD_BACKUP_ID, shardBackupId); + + shardRequestTracker.sendShardRequest(replica.getNodeName(), params, shardHandler); + log.debug("Sent backup request to core={} for backupName={}", coreName, backupName); + } + log.debug("Sent backup requests to all shard leaders for backupName={}", backupName); + + String msgOnError = "Could not backup all shards"; + shardRequestTracker.processResponses(results, shardHandler, true, msgOnError); + if (results.get("failure") != null) { + throw new SolrException(ErrorCode.SERVER_ERROR, msgOnError); + } + + //Aggregating result from different shards + @SuppressWarnings({"rawtypes"}) + NamedList aggRsp = aggregateResults(results, collectionName, backupManager, backupProperties, slices); + results.add("response", aggRsp); + } + + @SuppressWarnings({"rawtypes"}) + private NamedList aggregateResults(NamedList results, String collectionName, + BackupManager backupManager, + BackupProperties backupProps, + Collection slices) { + NamedList aggRsp = new NamedList<>(); + aggRsp.add("collection", collectionName); + aggRsp.add("numShards", slices.size()); + aggRsp.add("backupId", backupManager.getBackupId().id); + aggRsp.add("indexVersion", backupProps.getIndexVersion()); + aggRsp.add("startTime", backupProps.getStartTime()); + + double indexSizeMB = 0; + NamedList shards = (NamedList) results.get("success"); + for (int i = 0; i < shards.size(); i++) { + NamedList shardResp = (NamedList)((NamedList)shards.getVal(i)).get("response"); + if (shardResp == null) + continue; + indexSizeMB += (double) shardResp.get("indexSizeMB"); + } + aggRsp.add("indexSizeMB", indexSizeMB); + return aggRsp; + } + + private ModifiableSolrParams coreBackupParams(URI backupPath, String repoName, Slice slice, String coreName) { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString()); + params.set(NAME, slice.getName()); + params.set(CoreAdminParams.BACKUP_REPOSITORY, repoName); + params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); // note: index dir will be here then the "snapshot." + slice name + params.set(CORE_NAME_PROP, coreName); + return params; + } + @SuppressWarnings({"unchecked"}) - private void copyIndexFiles(URI backupPath, String collectionName, ZkNodeProps request, @SuppressWarnings({"rawtypes"})NamedList results) throws Exception { + private void copyIndexFiles(URI backupPath, String collectionName, ZkNodeProps request, @SuppressWarnings({"rawtypes"}) NamedList results) throws Exception { String backupName = request.getStr(NAME); String asyncId = request.getStr(ASYNC); String repoName = request.getStr(CoreAdminParams.BACKUP_REPOSITORY); @@ -179,16 +281,16 @@ private void copyIndexFiles(URI backupPath, String collectionName, ZkNodeProps r snapshotMeta = SolrSnapshotManager.getCollectionLevelSnapshot(zkClient, collectionName, commitName); if (!snapshotMeta.isPresent()) { throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName - + " does not exist for collection " + collectionName); + + " does not exist for collection " + collectionName); } if (snapshotMeta.get().getStatus() != SnapshotStatus.Successful) { throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName + " for collection " + collectionName - + " has not completed successfully. The status is " + snapshotMeta.get().getStatus()); + + " has not completed successfully. The status is " + snapshotMeta.get().getStatus()); } } log.info("Starting backup of collection={} with backupName={} at location={}", collectionName, backupName, - backupPath); + backupPath); Collection shardsToConsider = Collections.emptySet(); if (snapshotMeta.isPresent()) { @@ -202,7 +304,7 @@ private void copyIndexFiles(URI backupPath, String collectionName, ZkNodeProps r if (snapshotMeta.isPresent()) { if (!shardsToConsider.contains(slice.getName())) { log.warn("Skipping the backup for shard {} since it wasn't part of the collection {} when snapshot {} was created.", - slice.getName(), collectionName, snapshotMeta.get().getName()); + slice.getName(), collectionName, snapshotMeta.get().getName()); continue; } replica = selectReplicaWithSnapshot(snapshotMeta.get(), slice); @@ -216,12 +318,7 @@ private void copyIndexFiles(URI backupPath, String collectionName, ZkNodeProps r String coreName = replica.getStr(CORE_NAME_PROP); - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString()); - params.set(NAME, slice.getName()); - params.set(CoreAdminParams.BACKUP_REPOSITORY, repoName); - params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); // note: index dir will be here then the "snapshot." + slice name - params.set(CORE_NAME_PROP, coreName); + ModifiableSolrParams params = coreBackupParams(backupPath, repoName, slice, coreName); if (snapshotMeta.isPresent()) { params.set(CoreAdminParams.COMMIT_NAME, snapshotMeta.get().getName()); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java new file mode 100644 index 00000000000..04db6341579 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java @@ -0,0 +1,376 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud.api.collections; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URI; +import java.util.*; +import java.util.stream.Collectors; + +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.Pair; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.backup.BackupProperties; +import org.apache.solr.core.backup.BackupIdStats; +import org.apache.solr.core.backup.BackupId; +import org.apache.solr.core.backup.ShardBackupId; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.apache.solr.handler.IncrementalBackupPaths; + +import static org.apache.solr.common.params.CommonParams.NAME; +import static org.apache.solr.core.backup.BackupManager.COLLECTION_NAME_PROP; +import static org.apache.solr.core.backup.BackupManager.START_TIME_PROP; + +public class DeleteBackupCmd implements OverseerCollectionMessageHandler.Cmd { + private final OverseerCollectionMessageHandler ocmh; + + DeleteBackupCmd(OverseerCollectionMessageHandler ocmh) { + this.ocmh = ocmh; + } + + @Override + public void call(ClusterState state, ZkNodeProps message, @SuppressWarnings({"rawtypes"}) NamedList results) throws Exception { + String backupLocation = message.getStr(CoreAdminParams.BACKUP_LOCATION); + String backupName = message.getStr(NAME); + String repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY); + int backupId = message.getInt(CoreAdminParams.BACKUP_ID, -1); + int lastNumBackupPointsToKeep = message.getInt(CoreAdminParams.MAX_NUM_BACKUP, -1); + boolean purge = message.getBool(CoreAdminParams.PURGE_BACKUP, false); + if (backupId == -1 && lastNumBackupPointsToKeep == -1 && !purge) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, String.format(Locale.ROOT, "%s, %s or %s param must be provided", CoreAdminParams.BACKUP_ID, CoreAdminParams.MAX_NUM_BACKUP, + CoreAdminParams.PURGE_BACKUP)); + } + CoreContainer cc = ocmh.overseer.getCoreContainer(); + try (BackupRepository repository = cc.newBackupRepository(repo)) { + URI location = repository.createURI(backupLocation); + URI backupPath = repository.resolve(location, backupName); + + if (purge) { + purge(repository, backupPath, results); + } else if (backupId != -1){ + deleteBackupId(repository, backupPath, backupId, results); + } else { + keepNumberOfBackup(repository, backupPath, lastNumBackupPointsToKeep, results); + } + } + } + + @SuppressWarnings({"unchecked"}) + /** + * Clean up {@code backupPath} by removing all indexFiles, shardBackupIds, backupIds that are + * unreachable, uncompleted or corrupted. + */ + void purge(BackupRepository repository, URI backupPath, @SuppressWarnings({"rawtypes"}) NamedList result) throws IOException { + PurgeGraph purgeGraph = new PurgeGraph(); + purgeGraph.build(repository, backupPath); + + IncrementalBackupPaths backupPaths = new IncrementalBackupPaths(repository, backupPath); + repository.delete(backupPaths.getIndexDir(), purgeGraph.indexFileDeletes, true); + repository.delete(backupPaths.getShardBackupIdDir(), purgeGraph.shardBackupIdDeletes, true); + repository.delete(backupPath, purgeGraph.backupIdDeletes, true); + + @SuppressWarnings({"rawtypes"}) + NamedList details = new NamedList(); + details.add("numBackupIds", purgeGraph.backupIdDeletes.size()); + details.add("numShardBackupIds", purgeGraph.shardBackupIdDeletes.size()); + details.add("numIndexFiles", purgeGraph.indexFileDeletes.size()); + result.add("deleted", details); + } + + /** + * Keep most recent {@code maxNumBackup} and delete the rest. + */ + void keepNumberOfBackup(BackupRepository repository, URI backupPath, + int maxNumBackup, + @SuppressWarnings({"rawtypes"}) NamedList results) throws Exception { + List backupIds = BackupId.findAll(repository.listAllOrEmpty(backupPath)); + if (backupIds.size() <= maxNumBackup) { + return; + } + + Collections.sort(backupIds); + List backupIdDeletes = backupIds.subList(0, backupIds.size() - maxNumBackup); + deleteBackupIds(backupPath, repository, new HashSet<>(backupIdDeletes), results); + } + + void deleteBackupIds(URI backupPath, BackupRepository repository, + Set backupIdsDeletes, + @SuppressWarnings({"rawtypes"}) NamedList results) throws IOException { + IncrementalBackupPaths incBackupFiles = new IncrementalBackupPaths(repository, backupPath); + URI shardBackupIdDir = incBackupFiles.getShardBackupIdDir(); + + Set referencedIndexFiles = new HashSet<>(); + List> shardBackupIdDeletes = new ArrayList<>(); + + + String[] shardBackupIdFiles = repository.listAllOrEmpty(shardBackupIdDir); + for (String shardBackupIdFile : shardBackupIdFiles) { + Optional backupId = BackupProperties.backupIdOfShardBackupId(shardBackupIdFile); + if (!backupId.isPresent()) + continue; + + if (backupIdsDeletes.contains(backupId.get())) { + Pair pair = new Pair<>(backupId.get(), shardBackupIdFile); + shardBackupIdDeletes.add(pair); + } else { + ShardBackupId shardBackupId = ShardBackupId.from(repository, shardBackupIdDir, shardBackupIdFile); + if (shardBackupId != null) + referencedIndexFiles.addAll(shardBackupId.listUniqueFileNames()); + } + } + + + Map backupIdToCollectionBackupPoint = new HashMap<>(); + List unusedFiles = new ArrayList<>(); + for (Pair entry : shardBackupIdDeletes) { + BackupId backupId = entry.first(); + ShardBackupId shardBackupId = ShardBackupId.from(repository, shardBackupIdDir, entry.second()); + if (shardBackupId == null) + continue; + + backupIdToCollectionBackupPoint + .putIfAbsent(backupId, new BackupIdStats()); + backupIdToCollectionBackupPoint.get(backupId).add(shardBackupId); + + for (String uniqueIndexFile : shardBackupId.listUniqueFileNames()) { + if (!referencedIndexFiles.contains(uniqueIndexFile)) { + unusedFiles.add(uniqueIndexFile); + } + } + } + + repository.delete(incBackupFiles.getShardBackupIdDir(), + shardBackupIdDeletes.stream().map(Pair::second).collect(Collectors.toList()), true); + repository.delete(incBackupFiles.getIndexDir(), unusedFiles, true); + try { + for (BackupId backupId : backupIdsDeletes) { + repository.deleteDirectory(repository.resolve(backupPath, backupId.getZkStateDir())); + } + } catch (FileNotFoundException e) { + //ignore this + } + + //add details to result before deleting backupPropFiles + addResult(backupPath, repository, backupIdsDeletes, backupIdToCollectionBackupPoint, results); + repository.delete(backupPath, backupIdsDeletes.stream().map(BackupId::getBackupPropsName).collect(Collectors.toList()), true); + } + + @SuppressWarnings("unchecked") + private void addResult(URI backupPath, BackupRepository repository, + Set backupIdDeletes, + Map backupIdToCollectionBackupPoint, + @SuppressWarnings({"rawtypes"}) NamedList results) { + + String collectionName = null; + @SuppressWarnings({"rawtypes"}) + List shardBackupIdDetails = new ArrayList<>(); + results.add("deleted", shardBackupIdDetails); + for (BackupId backupId : backupIdDeletes) { + NamedList backupIdResult = new NamedList<>(); + + try { + BackupProperties props = BackupProperties.readFrom(repository, backupPath, backupId.getBackupPropsName()); + backupIdResult.add(START_TIME_PROP, props.getStartTime()); + if (collectionName == null) { + collectionName = props.getCollection(); + results.add(COLLECTION_NAME_PROP, collectionName); + } + } catch (IOException e) { + //prop file not found + } + + BackupIdStats cbp = backupIdToCollectionBackupPoint.getOrDefault(backupId, new BackupIdStats()); + backupIdResult.add("backupId", backupId.getId()); + backupIdResult.add("size", cbp.getTotalSize()); + backupIdResult.add("numFiles", cbp.getNumFiles()); + shardBackupIdDetails.add(backupIdResult); + } + } + + private void deleteBackupId(BackupRepository repository, URI backupPath, + int bid, @SuppressWarnings({"rawtypes"}) NamedList results) throws Exception { + BackupId backupId = new BackupId(bid); + if (!repository.exists(repository.resolve(backupPath, backupId.getBackupPropsName()))) { + return; + } + + deleteBackupIds(backupPath, repository, Collections.singleton(backupId), results); + } + + final static class PurgeGraph { + // graph + Map backupIdNodeMap = new HashMap<>(); + Map shardBackupIdNodeMap = new HashMap<>(); + Map indexFileNodeMap = new HashMap<>(); + + // delete queues + List backupIdDeletes = new ArrayList<>(); + List shardBackupIdDeletes = new ArrayList<>(); + List indexFileDeletes = new ArrayList<>(); + + public void build(BackupRepository repository, URI backupPath) throws IOException { + IncrementalBackupPaths backupPaths = new IncrementalBackupPaths(repository, backupPath); + buildLogicalGraph(repository, backupPath); + findDeletableNodes(repository, backupPaths); + } + + public void findDeletableNodes(BackupRepository repository, IncrementalBackupPaths backupPaths) { + // mark nodes as existing + visitExistingNodes(repository.listAllOrEmpty(backupPaths.getShardBackupIdDir()), + shardBackupIdNodeMap, shardBackupIdDeletes); + // this may be a long running commands + visitExistingNodes(repository.listAllOrEmpty(backupPaths.getIndexDir()), + indexFileNodeMap, indexFileDeletes); + + // for nodes which are not existing, propagate that information to other nodes + shardBackupIdNodeMap.values().forEach(Node::propagateNotExisting); + indexFileNodeMap.values().forEach(Node::propagateNotExisting); + + addDeleteNodesToQueue(backupIdNodeMap, backupIdDeletes); + addDeleteNodesToQueue(shardBackupIdNodeMap, shardBackupIdDeletes); + addDeleteNodesToQueue(indexFileNodeMap, indexFileDeletes); + } + + /** + * Visiting files (nodes) actually present in physical layer, + * if it does not present in the {@code nodeMap}, it should be deleted by putting into the {@code deleteQueue} + */ + private void visitExistingNodes(String[] existingNodeKeys, Map nodeMap, List deleteQueue) { + for (String nodeKey : existingNodeKeys) { + Node node = nodeMap.get(nodeKey); + + if (node == null) { + deleteQueue.add(nodeKey); + } else { + node.existing = true; + } + } + } + + private void addDeleteNodesToQueue(Map tNodeMap, List deleteQueue) { + tNodeMap.forEach((key, value) -> { + if (value.delete) { + deleteQueue.add(key); + } + }); + } + + Node getBackupIdNode(String backupPropsName) { + return backupIdNodeMap.computeIfAbsent(backupPropsName, bid -> { + Node node = new Node(); + node.existing = true; + return node; + }); + } + + Node getShardBackupIdNode(String shardBackupId) { + return shardBackupIdNodeMap.computeIfAbsent(shardBackupId, s -> new Node()); + } + + Node getIndexFileNode(String indexFile) { + return indexFileNodeMap.computeIfAbsent(indexFile, s -> new IndexFileNode()); + } + + void addEdge(Node node1, Node node2) { + node1.addNeighbor(node2); + node2.addNeighbor(node1); + } + + private void buildLogicalGraph(BackupRepository repository, URI backupPath) throws IOException { + List backupIds = BackupId.findAll(repository.listAllOrEmpty(backupPath)); + for (BackupId backupId : backupIds) { + BackupProperties backupProps = BackupProperties.readFrom(repository, backupPath, + backupId.getBackupPropsName()); + + Node backupIdNode = getBackupIdNode(backupId.getBackupPropsName()); + for (String shardBackupIdFile : backupProps.getAllShardBackupIdFiles()) { + Node shardBackupIdNode = getShardBackupIdNode(shardBackupIdFile); + addEdge(backupIdNode, shardBackupIdNode); + + ShardBackupId shardBackupId = ShardBackupId.from(repository, backupPath, shardBackupIdFile); + if (shardBackupId == null) + continue; + + for (String indexFile : shardBackupId.listUniqueFileNames()) { + Node indexFileNode = getIndexFileNode(indexFile); + addEdge(indexFileNode, shardBackupIdNode); + } + } + } + } + } + + //ShardBackupId, BackupId + static class Node { + List neighbors; + boolean delete = false; + boolean existing = false; + + void addNeighbor(Node node) { + if (neighbors == null) { + neighbors = new ArrayList<>(); + } + neighbors.add(node); + } + + void propagateNotExisting() { + if (existing) + return; + + if (neighbors != null) + neighbors.forEach(Node::propagateDelete); + } + + void propagateDelete() { + if (delete || !existing) + return; + + delete = true; + if (neighbors != null) { + neighbors.forEach(Node::propagateDelete); + } + } + } + + //IndexFile + final static class IndexFileNode extends Node { + int refCount = 0; + + @Override + void addNeighbor(Node node) { + super.addNeighbor(node); + refCount++; + } + + @Override + void propagateDelete() { + if (delete || !existing) + return; + + refCount--; + if (refCount == 0) { + delete = true; + } + } + } +} diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java index 42ee53d6599..347091a75a5 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -31,6 +32,7 @@ import java.util.concurrent.SynchronousQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; import com.google.common.collect.ImmutableMap; import org.apache.commons.lang3.StringUtils; @@ -78,6 +80,8 @@ import org.apache.solr.common.util.SuppressForbidden; import org.apache.solr.common.util.TimeSource; import org.apache.solr.common.util.Utils; +import org.apache.solr.core.backup.BackupId; +import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.handler.component.HttpShardHandlerFactory; import org.apache.solr.handler.component.ShardHandler; import org.apache.solr.handler.component.ShardRequest; @@ -200,6 +204,7 @@ public OverseerCollectionMessageHandler(ZkStateReader zkStateReader, String myId .put(DELETENODE, new DeleteNodeCmd(this)) .put(BACKUP, new BackupCmd(this)) .put(RESTORE, new RestoreCmd(this)) + .put(DELETEBACKUP, new DeleteBackupCmd(this)) .put(CREATESNAPSHOT, new CreateSnapshotCmd(this)) .put(DELETESNAPSHOT, new DeleteSnapshotCmd(this)) .put(SPLITSHARD, new SplitShardCmd(this)) @@ -576,7 +581,7 @@ void addPropertyParams(ZkNodeProps message, Map map) { } - private void modifyCollection(ClusterState clusterState, ZkNodeProps message, @SuppressWarnings({"rawtypes"})NamedList results) + void modifyCollection(ClusterState clusterState, ZkNodeProps message, @SuppressWarnings({"rawtypes"})NamedList results) throws Exception { final String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP); @@ -666,6 +671,19 @@ Map waitToSeeReplicasInState(String collectionName, Collection< } } + @SuppressWarnings({"rawtypes"}) + void cleanBackup(BackupRepository repository, URI backupPath, BackupId backupId) throws Exception { + ((DeleteBackupCmd)commandMap.get(DELETEBACKUP)) + .deleteBackupIds(backupPath, repository, Collections.singleton(backupId), new NamedList()); + } + + void deleteBackup(BackupRepository repository, URI backupPath, + int maxNumBackup, + @SuppressWarnings({"rawtypes"}) NamedList results) throws Exception { + ((DeleteBackupCmd)commandMap.get(DELETEBACKUP)) + .keepNumberOfBackup(repository, backupPath, maxNumBackup, results); + } + List addReplica(ClusterState clusterState, ZkNodeProps message, @SuppressWarnings({"rawtypes"})NamedList results, Runnable onComplete) throws Exception { @@ -920,6 +938,11 @@ public ShardRequestTracker asyncRequestTracker(String asyncId) { public class ShardRequestTracker{ private final String asyncId; private final NamedList shardAsyncIdByNode = new NamedList(); + private Consumer onResponseListener; + + public void setOnResponseListener(Consumer onResponseListener) { + this.onResponseListener = onResponseListener; + } private ShardRequestTracker(String asyncId) { this.asyncId = asyncId; @@ -986,6 +1009,9 @@ void processResponses(NamedList results, ShardHandler shardHandler, bool srsp = shardHandler.takeCompletedOrError(); if (srsp != null) { processResponse(results, srsp, okayExceptions); + if (onResponseListener != null) { + onResponseListener.accept(srsp); + } Throwable exception = srsp.getException(); if (abortOnError && exception != null) { // drain pending requests diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java index db408b4bf62..7cbf0d56b17 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java @@ -18,18 +18,11 @@ package org.apache.solr.cloud.api.collections; +import java.io.Closeable; +import java.io.IOException; import java.lang.invoke.MethodHandles; import java.net.URI; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Properties; -import java.util.Set; +import java.util.*; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -48,6 +41,7 @@ import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CollectionAdminParams; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; @@ -56,18 +50,14 @@ import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.backup.BackupManager; +import org.apache.solr.core.backup.BackupProperties; import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.handler.component.ShardHandler; +import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; -import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS; -import static org.apache.solr.common.cloud.ZkStateReader.PULL_REPLICAS; -import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; -import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE; -import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; -import static org.apache.solr.common.cloud.ZkStateReader.TLOG_REPLICAS; +import static org.apache.solr.common.cloud.ZkStateReader.*; import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE; import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD; import static org.apache.solr.common.params.CommonAdminParams.ASYNC; @@ -86,71 +76,134 @@ public RestoreCmd(OverseerCollectionMessageHandler ocmh) { @SuppressWarnings({"unchecked", "rawtypes"}) public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception { // TODO maybe we can inherit createCollection's options/code + try (RestoreContext restoreContext = new RestoreContext(message, ocmh)) { + if (state.hasCollection(restoreContext.restoreCollectionName)) { + RestoreOnExistingCollection restoreOnExistingCollection = new RestoreOnExistingCollection(restoreContext); + restoreOnExistingCollection.process(restoreContext, results); + } else { + RestoreOnANewCollection restoreOnANewCollection = new RestoreOnANewCollection(message, restoreContext.backupCollectionState); + restoreOnANewCollection.validate(restoreContext.backupCollectionState, restoreContext.nodeList.size()); + restoreOnANewCollection.process(results, restoreContext); + } + } + } - String restoreCollectionName = message.getStr(COLLECTION_PROP); - String backupName = message.getStr(NAME); // of backup - ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler(); - String asyncId = message.getStr(ASYNC); - String repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY); - - CoreContainer cc = ocmh.overseer.getCoreContainer(); - BackupRepository repository = cc.newBackupRepository(repo); - - URI location = repository.createURI(message.getStr(CoreAdminParams.BACKUP_LOCATION)); - URI backupPath = repository.resolve(location, backupName); - ZkStateReader zkStateReader = ocmh.zkStateReader; - BackupManager backupMgr = new BackupManager(repository, zkStateReader); - - Properties properties = backupMgr.readBackupProperties(location, backupName); - String backupCollection = properties.getProperty(BackupManager.COLLECTION_NAME_PROP); - - // Test if the collection is of stateFormat 1 (i.e. not 2) supported pre Solr 9, in which case can't restore it. - Object format = properties.get("stateFormat"); - if (format != null && !"2".equals(format)) { - throw new SolrException(ErrorCode.BAD_REQUEST, "Collection " + backupCollection + " is in stateFormat=" + format + - " no longer supported in Solr 9 and above. It can't be restored. If it originates in Solr 8 you can restore" + - " it there, migrate it to stateFormat=2 and backup again, it will then be restorable on Solr 9"); + private void requestReplicasToRestore(@SuppressWarnings({"rawtypes"}) NamedList results, DocCollection restoreCollection, + ClusterState clusterState, + BackupProperties backupProperties, + URI backupPath, + String repo, + ShardHandler shardHandler, + String asyncId) { + ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId); + // Copy data from backed up index to each replica + for (Slice slice : restoreCollection.getSlices()) { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.RESTORECORE.toString()); + Optional shardBackupId = backupProperties.getShardBackupIdFor(slice.getName()); + if (shardBackupId.isPresent()) { + params.set(CoreAdminParams.SHARD_BACKUP_ID, shardBackupId.get()); + } else { + params.set(NAME, "snapshot." + slice.getName()); + } + params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); + params.set(CoreAdminParams.BACKUP_REPOSITORY, repo); + shardRequestTracker.sliceCmd(clusterState, params, null, slice, shardHandler); } - String backupCollectionAlias = properties.getProperty(BackupManager.COLLECTION_ALIAS_PROP); - DocCollection backupCollectionState = backupMgr.readCollectionState(location, backupName, backupCollection); - - // Get the Solr nodes to restore a collection. - final List nodeList = Assign.getLiveOrLiveAndCreateNodeSetList( - zkStateReader.getClusterState().getLiveNodes(), message, OverseerCollectionMessageHandler.RANDOM); - - int numShards = backupCollectionState.getActiveSlices().size(); - - int numNrtReplicas; - if (message.get(REPLICATION_FACTOR) != null) { - numNrtReplicas = message.getInt(REPLICATION_FACTOR, 0); - } else if (message.get(NRT_REPLICAS) != null) { - numNrtReplicas = message.getInt(NRT_REPLICAS, 0); - } else { - //replicationFactor and nrtReplicas is always in sync after SOLR-11676 - //pick from cluster state of the backed up collection - numNrtReplicas = backupCollectionState.getReplicationFactor(); + shardRequestTracker.processResponses(new NamedList<>(), shardHandler, true, "Could not restore core"); + } + + private class RestoreOnANewCollection { + private int numNrtReplicas; + private int numTlogReplicas; + private int numPullReplicas; + private ZkNodeProps message; + + private RestoreOnANewCollection(ZkNodeProps message, DocCollection backupCollectionState) { + this.message = message; + + if (message.get(REPLICATION_FACTOR) != null) { + this.numNrtReplicas = message.getInt(REPLICATION_FACTOR, 0); + } else if (message.get(NRT_REPLICAS) != null) { + this.numNrtReplicas = message.getInt(NRT_REPLICAS, 0); + } else { + //replicationFactor and nrtReplicas is always in sync after SOLR-11676 + //pick from cluster state of the backed up collection + this.numNrtReplicas = backupCollectionState.getReplicationFactor(); + } + this.numTlogReplicas = getInt(message, TLOG_REPLICAS, backupCollectionState.getNumTlogReplicas(), 0); + this.numPullReplicas = getInt(message, PULL_REPLICAS, backupCollectionState.getNumPullReplicas(), 0); + } + + public void process(@SuppressWarnings("rawtypes") NamedList results, RestoreContext rc) throws Exception { + // Avoiding passing RestoreContext around + uploadConfig(rc.backupProperties.getConfigName(), + rc.restoreConfigName, + rc.zkStateReader, + rc.backupManager); + + log.info("Starting restore into collection={} with backup_name={} at location={}", rc.restoreCollectionName, rc.backupName, + rc.location); + createCoreLessCollection(rc.restoreCollectionName, + rc.restoreConfigName, + rc.backupCollectionState, + rc.zkStateReader.getClusterState()); + // note: when createCollection() returns, the collection exists (no race) + + // Restore collection properties + rc.backupManager.uploadCollectionProperties(rc.restoreCollectionName); + + DocCollection restoreCollection = rc.zkStateReader.getClusterState().getCollection(rc.restoreCollectionName); + markAllShardsAsConstruction(restoreCollection); + // TODO how do we leverage the RULE / SNITCH logic in createCollection? + ClusterState clusterState = rc.zkStateReader.getClusterState(); + + List sliceNames = new ArrayList<>(); + restoreCollection.getSlices().forEach(x -> sliceNames.add(x.getName())); + + List replicaPositions = getReplicaPositions(restoreCollection, rc.nodeList, clusterState, sliceNames); + + createSingleReplicaPerShard(results, restoreCollection, rc.asyncId, clusterState, replicaPositions); + Object failures = results.get("failure"); + if (failures != null && ((SimpleOrderedMap) failures).size() > 0) { + log.error("Restore failed to create initial replicas."); + ocmh.cleanupCollection(rc.restoreCollectionName, new NamedList<>()); + return; + } + + //refresh the location copy of collection state + restoreCollection = rc.zkStateReader.getClusterState().getCollection(rc.restoreCollectionName); + requestReplicasToRestore(results, restoreCollection, clusterState, rc.backupProperties, rc.backupPath, rc.repo, rc.shardHandler, rc.asyncId); + requestReplicasToApplyBufferUpdates(restoreCollection, rc.asyncId, rc.shardHandler); + markAllShardsAsActive(restoreCollection); + addReplicasToShards(results, clusterState, restoreCollection, replicaPositions, rc.asyncId); + restoringAlias(rc.backupProperties); + + log.info("Completed restoring collection={} backupName={}", restoreCollection, rc.backupName); + } - int numTlogReplicas = getInt(message, TLOG_REPLICAS, backupCollectionState.getNumTlogReplicas(), 0); - int numPullReplicas = getInt(message, PULL_REPLICAS, backupCollectionState.getNumPullReplicas(), 0); - int totalReplicasPerShard = numNrtReplicas + numTlogReplicas + numPullReplicas; - assert totalReplicasPerShard > 0; - - //Upload the configs - String configName = (String) properties.get(CollectionAdminParams.COLL_CONF); - String restoreConfigName = message.getStr(CollectionAdminParams.COLL_CONF, configName); - if (zkStateReader.getConfigManager().configExists(restoreConfigName)) { - log.info("Using existing config {}", restoreConfigName); - //TODO add overwrite option? - } else { - log.info("Uploading config {}", restoreConfigName); - backupMgr.uploadConfigDir(location, backupName, configName, restoreConfigName); + + private void validate(DocCollection backupCollectionState, int availableNodeCount) { + int numShards = backupCollectionState.getActiveSlices().size(); + int totalReplicasPerShard = numNrtReplicas + numTlogReplicas + numPullReplicas; + assert totalReplicasPerShard > 0; } - log.info("Starting restore into collection={} with backup_name={} at location={}", restoreCollectionName, backupName, - location); + private void uploadConfig(String configName, String restoreConfigName, ZkStateReader zkStateReader, BackupManager backupMgr) throws IOException { + if (zkStateReader.getConfigManager().configExists(restoreConfigName)) { + log.info("Using existing config {}", restoreConfigName); + //TODO add overwrite option? + } else { + log.info("Uploading config {}", restoreConfigName); + backupMgr.uploadConfigDir(configName, restoreConfigName); + } + } - //Create core-less collection - { + @SuppressWarnings({"unchecked", "rawtypes"}) + private void createCoreLessCollection(String restoreCollectionName, + String restoreConfigName, + DocCollection backupCollectionState, + ClusterState clusterState) throws Exception { Map propMap = new HashMap<>(); propMap.put(Overseer.QUEUE_OPERATION, CREATE.toString()); propMap.put("fromApi", "true"); // mostly true. Prevents autoCreated=true in the collection state. @@ -193,133 +246,105 @@ public void call(ClusterState state, ZkNodeProps message, NamedList results) thr propMap.put(OverseerCollectionMessageHandler.SHARDS_PROP, newSlices); } - ocmh.commandMap.get(CREATE).call(zkStateReader.getClusterState(), new ZkNodeProps(propMap), new NamedList()); + ocmh.commandMap.get(CREATE).call(clusterState, new ZkNodeProps(propMap), new NamedList()); // note: when createCollection() returns, the collection exists (no race) } - // Restore collection properties - backupMgr.uploadCollectionProperties(location, backupName, restoreCollectionName); - - DocCollection restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName); - //Mark all shards in CONSTRUCTION STATE while we restore the data - { + private void markAllShardsAsConstruction(DocCollection restoreCollection) throws KeeperException, InterruptedException { //TODO might instead createCollection accept an initial state? Is there a race? Map propMap = new HashMap<>(); propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower()); for (Slice shard : restoreCollection.getSlices()) { propMap.put(shard.getName(), Slice.State.CONSTRUCTION.toString()); } - propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollectionName); + propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollection.getName()); ocmh.overseer.offerStateUpdate(Utils.toJSON(new ZkNodeProps(propMap))); } - // TODO how do we leverage the RULE / SNITCH logic in createCollection? - - ClusterState clusterState = zkStateReader.getClusterState(); - - List sliceNames = new ArrayList<>(); - restoreCollection.getSlices().forEach(x -> sliceNames.add(x.getName())); - - Assign.AssignRequest assignRequest = new Assign.AssignRequestBuilder() - .forCollection(restoreCollectionName) - .forShard(sliceNames) - .assignNrtReplicas(numNrtReplicas) - .assignTlogReplicas(numTlogReplicas) - .assignPullReplicas(numPullReplicas) - .onNodes(nodeList) - .build(); - Assign.AssignStrategy assignStrategy = Assign.createAssignStrategy(ocmh.cloudManager, clusterState, restoreCollection); - List replicaPositions = assignStrategy.assign(ocmh.cloudManager, assignRequest); + private List getReplicaPositions(DocCollection restoreCollection, List nodeList, ClusterState clusterState, List sliceNames) throws IOException, InterruptedException { + Assign.AssignRequest assignRequest = new Assign.AssignRequestBuilder() + .forCollection(restoreCollection.getName()) + .forShard(sliceNames) + .assignNrtReplicas(numNrtReplicas) + .assignTlogReplicas(numTlogReplicas) + .assignPullReplicas(numPullReplicas) + .onNodes(nodeList) + .build(); + Assign.AssignStrategy assignStrategy = Assign.createAssignStrategy(ocmh.cloudManager, clusterState, restoreCollection); + return assignStrategy.assign(ocmh.cloudManager, assignRequest); + } - CountDownLatch countDownLatch = new CountDownLatch(restoreCollection.getSlices().size()); + @SuppressWarnings({"unchecked", "rawtypes"}) + private void createSingleReplicaPerShard(NamedList results, + DocCollection restoreCollection, + String asyncId, + ClusterState clusterState, List replicaPositions) throws Exception { + CountDownLatch countDownLatch = new CountDownLatch(restoreCollection.getSlices().size()); - //Create one replica per shard and copy backed up data to it - for (Slice slice : restoreCollection.getSlices()) { - if (log.isInfoEnabled()) { - log.info("Adding replica for shard={} collection={} ", slice.getName(), restoreCollection); - } - HashMap propMap = new HashMap<>(); - propMap.put(Overseer.QUEUE_OPERATION, CREATESHARD); - propMap.put(COLLECTION_PROP, restoreCollectionName); - propMap.put(SHARD_ID_PROP, slice.getName()); - - if (numNrtReplicas >= 1) { - propMap.put(REPLICA_TYPE, Replica.Type.NRT.name()); - } else if (numTlogReplicas >= 1) { - propMap.put(REPLICA_TYPE, Replica.Type.TLOG.name()); - } else { - throw new SolrException(ErrorCode.BAD_REQUEST, "Unexpected number of replicas, replicationFactor, " + - Replica.Type.NRT + " or " + Replica.Type.TLOG + " must be greater than 0"); - } - - // Get the first node matching the shard to restore in - String node; - for (ReplicaPosition replicaPosition : replicaPositions) { - if (Objects.equals(replicaPosition.shard, slice.getName())) { - node = replicaPosition.node; - propMap.put(CoreAdminParams.NODE, node); - replicaPositions.remove(replicaPosition); - break; + //Create one replica per shard and copy backed up data to it + for (Slice slice : restoreCollection.getSlices()) { + String sliceName = slice.getName(); + log.info("Adding replica for shard={} collection={} ", sliceName, restoreCollection); + HashMap propMap = new HashMap<>(); + propMap.put(Overseer.QUEUE_OPERATION, CREATESHARD); + propMap.put(COLLECTION_PROP, restoreCollection.getName()); + propMap.put(SHARD_ID_PROP, sliceName); + + if (numNrtReplicas >= 1) { + propMap.put(REPLICA_TYPE, Replica.Type.NRT.name()); + } else if (numTlogReplicas >= 1) { + propMap.put(REPLICA_TYPE, Replica.Type.TLOG.name()); + } else { + throw new SolrException(ErrorCode.BAD_REQUEST, "Unexpected number of replicas, replicationFactor, " + + Replica.Type.NRT + " or " + Replica.Type.TLOG + " must be greater than 0"); } - } - // add async param - if (asyncId != null) { - propMap.put(ASYNC, asyncId); - } - ocmh.addPropertyParams(message, propMap); - final NamedList addReplicaResult = new NamedList(); - ocmh.addReplica(clusterState, new ZkNodeProps(propMap), addReplicaResult, () -> { - Object addResultFailure = addReplicaResult.get("failure"); - if (addResultFailure != null) { - SimpleOrderedMap failure = (SimpleOrderedMap) results.get("failure"); - if (failure == null) { - failure = new SimpleOrderedMap(); - results.add("failure", failure); - } - failure.addAll((NamedList) addResultFailure); - } else { - SimpleOrderedMap success = (SimpleOrderedMap) results.get("success"); - if (success == null) { - success = new SimpleOrderedMap(); - results.add("success", success); + // Get the first node matching the shard to restore in + String node; + for (ReplicaPosition replicaPosition : replicaPositions) { + if (Objects.equals(replicaPosition.shard, sliceName)) { + node = replicaPosition.node; + propMap.put(CoreAdminParams.NODE, node); + replicaPositions.remove(replicaPosition); + break; } - success.addAll((NamedList) addReplicaResult.get("success")); } - countDownLatch.countDown(); - }); - } - boolean allIsDone = countDownLatch.await(1, TimeUnit.HOURS); - if (!allIsDone) { - throw new TimeoutException("Initial replicas were not created within 1 hour. Timing out."); - } - Object failures = results.get("failure"); - if (failures != null && ((SimpleOrderedMap) failures).size() > 0) { - log.error("Restore failed to create initial replicas."); - ocmh.cleanupCollection(restoreCollectionName, new NamedList()); - return; - } - - //refresh the location copy of collection state - restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName); + // add async param + if (asyncId != null) { + propMap.put(ASYNC, asyncId); + } + ocmh.addPropertyParams(message, propMap); + final NamedList addReplicaResult = new NamedList(); + ocmh.addReplica(clusterState, new ZkNodeProps(propMap), addReplicaResult, () -> { + Object addResultFailure = addReplicaResult.get("failure"); + if (addResultFailure != null) { + SimpleOrderedMap failure = (SimpleOrderedMap) results.get("failure"); + if (failure == null) { + failure = new SimpleOrderedMap(); + results.add("failure", failure); + } + failure.addAll((NamedList) addResultFailure); + } else { + SimpleOrderedMap success = (SimpleOrderedMap) results.get("success"); + if (success == null) { + success = new SimpleOrderedMap(); + results.add("success", success); + } + success.addAll((NamedList) addReplicaResult.get("success")); + } + countDownLatch.countDown(); + }); + } - { - ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId); - // Copy data from backed up index to each replica - for (Slice slice : restoreCollection.getSlices()) { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.RESTORECORE.toString()); - params.set(NAME, "snapshot." + slice.getName()); - params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); - params.set(CoreAdminParams.BACKUP_REPOSITORY, repo); - shardRequestTracker.sliceCmd(clusterState, params, null, slice, shardHandler); + boolean allIsDone = countDownLatch.await(1, TimeUnit.HOURS); + if (!allIsDone) { + throw new TimeoutException("Initial replicas were not created within 1 hour. Timing out."); } - shardRequestTracker.processResponses(new NamedList(), shardHandler, true, "Could not restore core"); } - { + private void requestReplicasToApplyBufferUpdates(DocCollection restoreCollection, String asyncId, ShardHandler shardHandler) { ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId); for (Slice s : restoreCollection.getSlices()) { @@ -328,10 +353,7 @@ public void call(ClusterState state, ZkNodeProps message, NamedList results) thr String coreNodeName = r.getCoreName(); Replica.State stateRep = r.getState(); - if (log.isDebugEnabled()) { - log.debug("Calling REQUESTAPPLYUPDATES on: nodeName={}, coreNodeName={}, state={}", nodeName, coreNodeName, - stateRep.name()); - } + log.debug("Calling REQUESTAPPLYUPDATES on: nodeName={}, coreNodeName={}, state={}", nodeName, coreNodeName, stateRep); ModifiableSolrParams params = new ModifiableSolrParams(); params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.REQUESTAPPLYUPDATES.toString()); @@ -340,94 +362,203 @@ public void call(ClusterState state, ZkNodeProps message, NamedList results) thr shardRequestTracker.sendShardRequest(nodeName, params, shardHandler); } - shardRequestTracker.processResponses(new NamedList(), shardHandler, true, + shardRequestTracker.processResponses(new NamedList<>(), shardHandler, true, "REQUESTAPPLYUPDATES calls did not succeed"); } } //Mark all shards in ACTIVE STATE - { + private void markAllShardsAsActive(DocCollection restoreCollection) throws KeeperException, InterruptedException { HashMap propMap = new HashMap<>(); propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower()); - propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollectionName); + propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollection.getName()); for (Slice shard : restoreCollection.getSlices()) { propMap.put(shard.getName(), Slice.State.ACTIVE.toString()); } ocmh.overseer.offerStateUpdate((Utils.toJSON(new ZkNodeProps(propMap)))); } - if (totalReplicasPerShard > 1) { - if (log.isInfoEnabled()) { - log.info("Adding replicas to restored collection={}", restoreCollection.getName()); - } - for (Slice slice : restoreCollection.getSlices()) { - - //Add the remaining replicas for each shard, considering it's type - int createdNrtReplicas = 0, createdTlogReplicas = 0, createdPullReplicas = 0; - - // We already created either a NRT or an TLOG replica as leader - if (numNrtReplicas > 0) { - createdNrtReplicas++; - } else if (createdTlogReplicas > 0) { - createdTlogReplicas++; + private void addReplicasToShards(@SuppressWarnings({"rawtypes"}) NamedList results, + ClusterState clusterState, + DocCollection restoreCollection, + List replicaPositions, + String asyncId) throws Exception { + int totalReplicasPerShard = numNrtReplicas + numTlogReplicas + numPullReplicas; + if (totalReplicasPerShard > 1) { + if (log.isInfoEnabled()) { + log.info("Adding replicas to restored collection={}", restoreCollection.getName()); } + for (Slice slice : restoreCollection.getSlices()) { - for (int i = 1; i < totalReplicasPerShard; i++) { - Replica.Type typeToCreate; - if (createdNrtReplicas < numNrtReplicas) { + //Add the remaining replicas for each shard, considering it's type + int createdNrtReplicas = 0, createdTlogReplicas = 0, createdPullReplicas = 0; + + // We already created either a NRT or an TLOG replica as leader + if (numNrtReplicas > 0) { createdNrtReplicas++; - typeToCreate = Replica.Type.NRT; - } else if (createdTlogReplicas < numTlogReplicas) { + } else if (numTlogReplicas > 0) { createdTlogReplicas++; - typeToCreate = Replica.Type.TLOG; - } else { - createdPullReplicas++; - typeToCreate = Replica.Type.PULL; - assert createdPullReplicas <= numPullReplicas : "Unexpected number of replicas"; } - if (log.isDebugEnabled()) { - log.debug("Adding replica for shard={} collection={} of type {} ", slice.getName(), restoreCollection, typeToCreate); - } - HashMap propMap = new HashMap<>(); - propMap.put(COLLECTION_PROP, restoreCollectionName); - propMap.put(SHARD_ID_PROP, slice.getName()); - propMap.put(REPLICA_TYPE, typeToCreate.name()); - - // Get the first node matching the shard to restore in - String node; - for (ReplicaPosition replicaPosition : replicaPositions) { - if (Objects.equals(replicaPosition.shard, slice.getName())) { - node = replicaPosition.node; - propMap.put(CoreAdminParams.NODE, node); - replicaPositions.remove(replicaPosition); - break; + for (int i = 1; i < totalReplicasPerShard; i++) { + Replica.Type typeToCreate; + if (createdNrtReplicas < numNrtReplicas) { + createdNrtReplicas++; + typeToCreate = Replica.Type.NRT; + } else if (createdTlogReplicas < numTlogReplicas) { + createdTlogReplicas++; + typeToCreate = Replica.Type.TLOG; + } else { + createdPullReplicas++; + typeToCreate = Replica.Type.PULL; + assert createdPullReplicas <= numPullReplicas : "Unexpected number of replicas"; } - } - // add async param - if (asyncId != null) { - propMap.put(ASYNC, asyncId); - } - ocmh.addPropertyParams(message, propMap); + if (log.isDebugEnabled()) { + log.debug("Adding replica for shard={} collection={} of type {} ", slice.getName(), restoreCollection, typeToCreate); + } + HashMap propMap = new HashMap<>(); + propMap.put(COLLECTION_PROP, restoreCollection.getName()); + propMap.put(SHARD_ID_PROP, slice.getName()); + propMap.put(REPLICA_TYPE, typeToCreate.name()); + + // Get the first node matching the shard to restore in + String node; + for (ReplicaPosition replicaPosition : replicaPositions) { + if (Objects.equals(replicaPosition.shard, slice.getName())) { + node = replicaPosition.node; + propMap.put(CoreAdminParams.NODE, node); + replicaPositions.remove(replicaPosition); + break; + } + } + + // add async param + if (asyncId != null) { + propMap.put(ASYNC, asyncId); + } + ocmh.addPropertyParams(message, propMap); - ocmh.addReplica(zkStateReader.getClusterState(), new ZkNodeProps(propMap), results, null); + ocmh.addReplica(clusterState, new ZkNodeProps(propMap), results, null); + } } } } - if (backupCollectionAlias != null && !backupCollectionAlias.equals(backupCollection)) { - log.debug("Restoring alias {} -> {}", backupCollectionAlias, backupCollection); - ocmh.zkStateReader.aliasesManager - .applyModificationAndExportToZk(a -> a.cloneWithCollectionAlias(backupCollectionAlias, backupCollection)); + private void restoringAlias(BackupProperties properties) { + String backupCollection = properties.getCollection(); + String backupCollectionAlias = properties.getCollectionAlias(); + if (backupCollectionAlias != null && !backupCollectionAlias.equals(backupCollection)) { + log.debug("Restoring alias {} -> {}", backupCollectionAlias, backupCollection); + ocmh.zkStateReader.aliasesManager + .applyModificationAndExportToZk(a -> a.cloneWithCollectionAlias(backupCollectionAlias, backupCollection)); + } } - - log.info("Completed restoring collection={} backupName={}", restoreCollection, backupName); - } private int getInt(ZkNodeProps message, String propertyName, Integer count, int defaultValue) { Integer value = message.getInt(propertyName, count); return value!=null ? value:defaultValue; } + + private static class RestoreContext implements Closeable { + + final String restoreCollectionName; + final String backupName; + final String backupCollection; + final String asyncId; + final String repo; + final String restoreConfigName; + final int backupId; + final URI location; + final URI backupPath; + final List nodeList; + + final CoreContainer container; + final BackupRepository repository; + final ZkStateReader zkStateReader; + final BackupManager backupManager; + final BackupProperties backupProperties; + final DocCollection backupCollectionState; + final ShardHandler shardHandler; + + private RestoreContext(ZkNodeProps message, OverseerCollectionMessageHandler ocmh) throws IOException { + this.restoreCollectionName = message.getStr(COLLECTION_PROP); + this.backupName = message.getStr(NAME); // of backup + this.asyncId = message.getStr(ASYNC); + this.repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY); + this.backupId = message.getInt(CoreAdminParams.BACKUP_ID, -1); + + this.container = ocmh.overseer.getCoreContainer(); + this.repository = this.container.newBackupRepository(repo); + + this.location = repository.createURI(message.getStr(CoreAdminParams.BACKUP_LOCATION)); + this.backupPath = repository.resolve(location, backupName); + this.zkStateReader = ocmh.zkStateReader; + this.backupManager = backupId == -1 ? + BackupManager.forRestore(repository, zkStateReader, backupPath) : + BackupManager.forRestore(repository, zkStateReader, backupPath, backupId); + + this.backupProperties = this.backupManager.readBackupProperties(); + this.backupCollection = this.backupProperties.getCollection(); + this.restoreConfigName = message.getStr(CollectionAdminParams.COLL_CONF, this.backupProperties.getConfigName()); + this.backupCollectionState = this.backupManager.readCollectionState(this.backupCollection); + + this.shardHandler = ocmh.shardHandlerFactory.getShardHandler(); + this.nodeList = Assign.getLiveOrLiveAndCreateNodeSetList( + zkStateReader.getClusterState().getLiveNodes(), message, OverseerCollectionMessageHandler.RANDOM); + } + + @Override + public void close() throws IOException { + if (this.repository != null) { + this.repository.close(); + } + } + } + + private class RestoreOnExistingCollection { + + private RestoreOnExistingCollection(RestoreContext rc) { + int numShardsOfBackup = rc.backupCollectionState.getSlices().size(); + int numShards = rc.zkStateReader.getClusterState().getCollection(rc.restoreCollectionName).getSlices().size(); + + if (numShardsOfBackup != numShards) { + String msg = String.format(Locale.ROOT, "Unable to restoring since number of shards in backup " + + "and specified collection does not match, numShardsOfBackup:%d numShardsOfCollection:%d", numShardsOfBackup, numShards); + throw new SolrException(ErrorCode.BAD_REQUEST, msg); + } + } + + public void process(RestoreContext rc, @SuppressWarnings({"rawtypes"}) NamedList results) throws Exception { + ClusterState clusterState = rc.zkStateReader.getClusterState(); + DocCollection restoreCollection = clusterState.getCollection(rc.restoreCollectionName); + + enableReadOnly(clusterState, restoreCollection); + try { + requestReplicasToRestore(results, restoreCollection, clusterState, rc.backupProperties, + rc.backupPath, rc.repo, rc.shardHandler, rc.asyncId); + } finally { + disableReadOnly(clusterState, restoreCollection); + } + } + + private void disableReadOnly(ClusterState clusterState, DocCollection restoreCollection) throws Exception { + ZkNodeProps params = new ZkNodeProps( + Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.MODIFYCOLLECTION.toString(), + ZkStateReader.COLLECTION_PROP, restoreCollection.getName(), + ZkStateReader.READ_ONLY, null + ); + ocmh.modifyCollection(clusterState, params, new NamedList<>()); + } + + private void enableReadOnly(ClusterState clusterState, DocCollection restoreCollection) throws Exception { + ZkNodeProps params = new ZkNodeProps( + Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.MODIFYCOLLECTION.toString(), + ZkStateReader.COLLECTION_PROP, restoreCollection.getName(), + ZkStateReader.READ_ONLY, "true" + ); + ocmh.modifyCollection(clusterState, params, new NamedList<>()); + } + } } diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupId.java b/solr/core/src/java/org/apache/solr/core/backup/BackupId.java new file mode 100644 index 00000000000..6d5d788ba1e --- /dev/null +++ b/solr/core/src/java/org/apache/solr/core/backup/BackupId.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.core.backup; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.apache.solr.core.backup.BackupManager.ZK_STATE_DIR; + +public class BackupId implements Comparable{ + private static final Pattern BACKUP_PROPS_ID_PTN = Pattern.compile("backup_([0-9]+).properties"); + + public final int id; + + public BackupId(int id) { + this.id = id; + } + + public static BackupId zero() { + return new BackupId(0); + } + + public static BackupId oldVersion() { + return new BackupId(-1); + } + + public BackupId nextBackupId() { + return new BackupId(id+1); + } + + public static List findAll(String[] listFiles) { + List result = new ArrayList<>(); + for (String file: listFiles) { + Matcher m = BACKUP_PROPS_ID_PTN.matcher(file); + if (m.find()) { + result.add(new BackupId(Integer.parseInt(m.group(1)))); + } + } + + return result; + } + + public String getZkStateDir() { + if (id == -1) { + return ZK_STATE_DIR; + } + return String.format(Locale.ROOT, "%s_%d/", ZK_STATE_DIR, id); + } + + public String getBackupPropsName() { + if (id == -1) { + return BackupManager.BACKUP_PROPS_FILE; + } + return getBackupPropsName(id); + } + + private static String getBackupPropsName(int id) { + return String.format(Locale.ROOT, "backup_%d.properties", id); + } + + static Optional findMostRecent(String[] listFiles) { + return findAll(listFiles).stream().max(Comparator.comparingInt(o -> o.id)); + } + + public int getId() { + return id; + } + + @Override + public int compareTo(BackupId o) { + return Integer.compare(this.id, o.id); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BackupId backupId = (BackupId) o; + return id == backupId.id; + } + + @Override + public int hashCode() { + return id; + } +} diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupIdStats.java b/solr/core/src/java/org/apache/solr/core/backup/BackupIdStats.java new file mode 100644 index 00000000000..cb38665fb9f --- /dev/null +++ b/solr/core/src/java/org/apache/solr/core/backup/BackupIdStats.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.core.backup; + +/** + * Aggregate stats from multiple {@link ShardBackupId} + */ +public class BackupIdStats { + private int numFiles = 0; + private long totalSize = 0; + + public BackupIdStats() { + } + + public void add(ShardBackupId shardBackupId) { + numFiles += shardBackupId.numFiles(); + totalSize += shardBackupId.totalSize(); + } + + public int getNumFiles() { + return numFiles; + } + + public long getTotalSize() { + return totalSize; + } + +} diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java index deae3604c79..3a52f2210d1 100644 --- a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java +++ b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java @@ -17,10 +17,8 @@ package org.apache.solr.core.backup; import java.io.IOException; -import java.io.InputStreamReader; import java.io.OutputStream; import java.io.OutputStreamWriter; -import java.io.Reader; import java.io.Writer; import java.lang.invoke.MethodHandles; import java.net.URI; @@ -28,7 +26,7 @@ import java.util.Collections; import java.util.List; import java.util.Objects; -import java.util.Properties; +import java.util.Optional; import com.google.common.base.Preconditions; import org.apache.lucene.store.IOContext; @@ -43,7 +41,6 @@ import org.apache.solr.common.util.Utils; import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.core.backup.repository.BackupRepository.PathType; -import org.apache.solr.util.PropertiesInputStream; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; @@ -64,15 +61,92 @@ public class BackupManager { public static final String COLLECTION_NAME_PROP = "collection"; public static final String COLLECTION_ALIAS_PROP = "collectionAlias"; public static final String BACKUP_NAME_PROP = "backupName"; - public static final String INDEX_VERSION_PROP = "index.version"; + public static final String INDEX_VERSION_PROP = "indexVersion"; public static final String START_TIME_PROP = "startTime"; protected final ZkStateReader zkStateReader; protected final BackupRepository repository; + protected final BackupId backupId; + protected final URI backupPath; + protected final String existingPropsFile; - public BackupManager(BackupRepository repository, ZkStateReader zkStateReader) { + + private BackupManager(BackupRepository repository, + URI backupPath, + ZkStateReader zkStateReader, + String existingPropsFile, + BackupId backupId) { this.repository = Objects.requireNonNull(repository); + this.backupPath = backupPath; this.zkStateReader = Objects.requireNonNull(zkStateReader); + this.existingPropsFile = existingPropsFile; + this.backupId = backupId; + } + + public static BackupManager forIncrementalBackup(BackupRepository repository, + ZkStateReader stateReader, + URI backupPath) { + Objects.requireNonNull(repository); + Objects.requireNonNull(stateReader); + + Optional lastBackupId = BackupId.findMostRecent(repository.listAllOrEmpty(backupPath)); + + return new BackupManager(repository, backupPath, stateReader, lastBackupId + .map(BackupId::getBackupPropsName).orElse(null), + lastBackupId.map(BackupId::nextBackupId).orElse(BackupId.zero())); + } + + public static BackupManager forBackup(BackupRepository repository, + ZkStateReader stateReader, + URI backupLoc, + String backupName) { + Objects.requireNonNull(repository); + Objects.requireNonNull(stateReader); + + URI backupPath = repository.resolve(backupLoc, backupName); + return new BackupManager(repository, backupPath, stateReader, null, BackupId.oldVersion()); + } + + public static BackupManager forRestore(BackupRepository repository, + ZkStateReader stateReader, + URI backupPath, + int bid) throws IOException { + Objects.requireNonNull(repository); + Objects.requireNonNull(stateReader); + + BackupId backupId = new BackupId(bid); + String backupPropsName = backupId.getBackupPropsName(); + if (!repository.exists(repository.resolve(backupPath, backupPropsName))) { + throw new IllegalStateException("Backup id " + bid + " was not found"); + } + + return new BackupManager(repository, backupPath, stateReader, backupPropsName, backupId); + } + + public static BackupManager forRestore(BackupRepository repository, + ZkStateReader stateReader, + URI backupPath) throws IOException { + Objects.requireNonNull(repository); + Objects.requireNonNull(stateReader); + + if (!repository.exists(backupPath)) { + throw new SolrException(ErrorCode.SERVER_ERROR, "Couldn't restore since doesn't exist: " + backupPath); + } + + Optional opFileGen = BackupId.findMostRecent(repository.listAll(backupPath)); + if (opFileGen.isPresent()) { + BackupId backupPropFile = opFileGen.get(); + return new BackupManager(repository, backupPath, stateReader, backupPropFile.getBackupPropsName(), + backupPropFile); + } else if (repository.exists(repository.resolve(backupPath, BACKUP_PROPS_FILE))){ + return new BackupManager(repository, backupPath, stateReader, BACKUP_PROPS_FILE, null); + } else { + throw new IllegalStateException("No " + BACKUP_PROPS_FILE + " was found, the backup does not exist or not complete"); + } + } + + public final BackupId getBackupId() { + return backupId; } /** @@ -85,57 +159,49 @@ public final String getVersion() { /** * This method returns the configuration parameters for the specified backup. * - * @param backupLoc The base path used to store the backup data. - * @param backupId The unique name for the backup whose configuration params are required. * @return the configuration parameters for the specified backup. * @throws IOException In case of errors. */ - public Properties readBackupProperties(URI backupLoc, String backupId) throws IOException { - Objects.requireNonNull(backupLoc); - Objects.requireNonNull(backupId); - - // Backup location - URI backupPath = repository.resolve(backupLoc, backupId); - if (!repository.exists(backupPath)) { - throw new SolrException(ErrorCode.SERVER_ERROR, "Couldn't restore since doesn't exist: " + backupPath); + public BackupProperties readBackupProperties() throws IOException { + if (existingPropsFile == null) { + throw new IllegalStateException("No " + BACKUP_PROPS_FILE + " was found, the backup does not exist or not complete"); } - Properties props = new Properties(); - try (Reader is = new InputStreamReader(new PropertiesInputStream( - repository.openInput(backupPath, BACKUP_PROPS_FILE, IOContext.DEFAULT)), StandardCharsets.UTF_8)) { - props.load(is); - return props; + return BackupProperties.readFrom(repository, backupPath, existingPropsFile); + } + + public Optional tryReadBackupProperties() throws IOException { + if (existingPropsFile != null) { + return Optional.of(BackupProperties.readFrom(repository, backupPath, existingPropsFile)); } + + return Optional.empty(); } /** * This method stores the backup properties at the specified location in the repository. * - * @param backupLoc The base path used to store the backup data. - * @param backupId The unique name for the backup whose configuration params are required. * @param props The backup properties * @throws IOException in case of I/O error */ - public void writeBackupProperties(URI backupLoc, String backupId, Properties props) throws IOException { - URI dest = repository.resolve(backupLoc, backupId, BACKUP_PROPS_FILE); + public void writeBackupProperties(BackupProperties props) throws IOException { + URI dest = repository.resolve(backupPath, backupId.getBackupPropsName()); try (Writer propsWriter = new OutputStreamWriter(repository.createOutput(dest), StandardCharsets.UTF_8)) { - props.store(propsWriter, "Backup properties file"); + props.store(propsWriter); } } /** * This method reads the meta-data information for the backed-up collection. * - * @param backupLoc The base path used to store the backup data. - * @param backupId The unique name for the backup. * @param collectionName The name of the collection whose meta-data is to be returned. * @return the meta-data information for the backed-up collection. * @throws IOException in case of errors. */ - public DocCollection readCollectionState(URI backupLoc, String backupId, String collectionName) throws IOException { + public DocCollection readCollectionState(String collectionName) throws IOException { Objects.requireNonNull(collectionName); - URI zkStateDir = repository.resolve(backupLoc, backupId, ZK_STATE_DIR); + URI zkStateDir = getZkStateDir(); try (IndexInput is = repository.openInput(zkStateDir, COLLECTION_PROPS_FILE, IOContext.DEFAULT)) { byte[] arr = new byte[(int) is.length()]; // probably ok since the json file should be small. is.readBytes(arr, 0, (int) is.length()); @@ -147,15 +213,13 @@ public DocCollection readCollectionState(URI backupLoc, String backupId, String /** * This method writes the collection meta-data to the specified location in the repository. * - * @param backupLoc The base path used to store the backup data. - * @param backupId The unique name for the backup. * @param collectionName The name of the collection whose meta-data is being stored. * @param collectionState The collection meta-data to be stored. * @throws IOException in case of I/O errors. */ - public void writeCollectionState(URI backupLoc, String backupId, String collectionName, + public void writeCollectionState(String collectionName, DocCollection collectionState) throws IOException { - URI dest = repository.resolve(backupLoc, backupId, ZK_STATE_DIR, COLLECTION_PROPS_FILE); + URI dest = getZkStateDir(COLLECTION_PROPS_FILE); try (OutputStream collectionStateOs = repository.createOutput(dest)) { collectionStateOs.write(Utils.toJSON(Collections.singletonMap(collectionName, collectionState))); } @@ -164,38 +228,38 @@ public void writeCollectionState(URI backupLoc, String backupId, String collecti /** * This method uploads the Solr configuration files to the desired location in Zookeeper. * - * @param backupLoc The base path used to store the backup data. - * @param backupId The unique name for the backup. * @param sourceConfigName The name of the config to be copied * @param targetConfigName The name of the config to be created. * @throws IOException in case of I/O errors. */ - public void uploadConfigDir(URI backupLoc, String backupId, String sourceConfigName, String targetConfigName) - throws IOException { - URI source = repository.resolve(backupLoc, backupId, ZK_STATE_DIR, CONFIG_STATE_DIR, sourceConfigName); + public void uploadConfigDir(String sourceConfigName, String targetConfigName) + throws IOException { + URI source = getZkStateDir(CONFIG_STATE_DIR, sourceConfigName); String zkPath = ZkConfigManager.CONFIGS_ZKNODE + "/" + targetConfigName; + + Preconditions.checkState(repository.exists(source), "Path {} does not exist", source); + Preconditions.checkState(repository.getPathType(source) == PathType.DIRECTORY, + "Path {} is not a directory", zkPath); uploadToZk(zkStateReader.getZkClient(), source, zkPath); } /** * This method stores the contents of a specified Solr config at the specified location in repository. * - * @param backupLoc The base path used to store the backup data. - * @param backupId The unique name for the backup. * @param configName The name of the config to be saved. * @throws IOException in case of I/O errors. */ - public void downloadConfigDir(URI backupLoc, String backupId, String configName) throws IOException { - URI dest = repository.resolve(backupLoc, backupId, ZK_STATE_DIR, CONFIG_STATE_DIR, configName); - repository.createDirectory(repository.resolve(backupLoc, backupId, ZK_STATE_DIR)); - repository.createDirectory(repository.resolve(backupLoc, backupId, ZK_STATE_DIR, CONFIG_STATE_DIR)); + public void downloadConfigDir(String configName) throws IOException { + URI dest = getZkStateDir(CONFIG_STATE_DIR, configName); + repository.createDirectory(getZkStateDir()); + repository.createDirectory(getZkStateDir(CONFIG_STATE_DIR)); repository.createDirectory(dest); downloadFromZK(zkStateReader.getZkClient(), ZkConfigManager.CONFIGS_ZKNODE + "/" + configName, dest); } - public void uploadCollectionProperties(URI backupLoc, String backupId, String collectionName) throws IOException { - URI sourceDir = repository.resolve(backupLoc, backupId, ZK_STATE_DIR); + public void uploadCollectionProperties(String collectionName) throws IOException { + URI sourceDir = getZkStateDir(); URI source = repository.resolve(sourceDir, ZkStateReader.COLLECTION_PROPS_ZKNODE); if (!repository.exists(source)) { // No collection properties to restore @@ -209,12 +273,12 @@ public void uploadCollectionProperties(URI backupLoc, String backupId, String co zkStateReader.getZkClient().create(zkPath, arr, CreateMode.PERSISTENT, true); } catch (KeeperException | InterruptedException e) { throw new IOException("Error uploading file to zookeeper path " + source.toString() + " to " + zkPath, - SolrZkClient.checkInterrupted(e)); + SolrZkClient.checkInterrupted(e)); } } - public void downloadCollectionProperties(URI backupLoc, String backupId, String collectionName) throws IOException { - URI dest = repository.resolve(backupLoc, backupId, ZK_STATE_DIR, ZkStateReader.COLLECTION_PROPS_ZKNODE); + public void downloadCollectionProperties(String collectionName) throws IOException { + URI dest = getZkStateDir(ZkStateReader.COLLECTION_PROPS_ZKNODE); String zkPath = ZkStateReader.COLLECTIONS_ZKNODE + '/' + collectionName + '/' + ZkStateReader.COLLECTION_PROPS_ZKNODE; @@ -230,15 +294,12 @@ public void downloadCollectionProperties(URI backupLoc, String backupId, String } } catch (KeeperException | InterruptedException e) { throw new IOException("Error downloading file from zookeeper path " + zkPath + " to " + dest.toString(), - SolrZkClient.checkInterrupted(e)); + SolrZkClient.checkInterrupted(e)); } } private void downloadFromZK(SolrZkClient zkClient, String zkPath, URI dir) throws IOException { try { - if (!repository.exists(dir)) { - repository.createDirectory(dir); - } List files = zkClient.getChildren(zkPath, null, true); for (String file : files) { List children = zkClient.getChildren(zkPath + "/" + file, null, true); @@ -249,20 +310,20 @@ private void downloadFromZK(SolrZkClient zkClient, String zkPath, URI dir) throw os.write(data); } } else { + URI uri = repository.resolve(dir, file); + if (!repository.exists(uri)) { + repository.createDirectory(uri); + } downloadFromZK(zkClient, zkPath + "/" + file, repository.resolve(dir, file)); } } } catch (KeeperException | InterruptedException e) { throw new IOException("Error downloading files from zookeeper path " + zkPath + " to " + dir.toString(), - SolrZkClient.checkInterrupted(e)); + SolrZkClient.checkInterrupted(e)); } } private void uploadToZk(SolrZkClient zkClient, URI sourceDir, String destZkPath) throws IOException { - Preconditions.checkArgument(repository.exists(sourceDir), "Path {} does not exist", sourceDir); - Preconditions.checkArgument(repository.getPathType(sourceDir) == PathType.DIRECTORY, - "Path {} is not a directory", sourceDir); - for (String file : repository.listAll(sourceDir)) { String zkNodePath = destZkPath + "/" + file; URI path = repository.resolve(sourceDir, file); @@ -290,4 +351,19 @@ private void uploadToZk(SolrZkClient zkClient, URI sourceDir, String destZkPath) } } } + + private URI getZkStateDir(String... subFolders) { + URI zkStateDir; + if (backupId != null) { + String zkBackupFolder = backupId.getZkStateDir(); + zkStateDir = repository.resolve(backupPath, zkBackupFolder); + } else { + zkStateDir = repository.resolve(backupPath, ZK_STATE_DIR); + } + + if (subFolders.length == 0) { + return zkStateDir; + } + return repository.resolve(zkStateDir, subFolders); + } } diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.java b/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.java new file mode 100644 index 00000000000..d797fdc7d93 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.core.backup; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.io.Writer; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +import org.apache.lucene.store.IOContext; +import org.apache.lucene.util.Version; +import org.apache.solr.common.params.CollectionAdminParams; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.apache.solr.util.PropertiesInputStream; + +/** + * Represent a backup-*.properties file. + * As well as providing methods for reading/writing these files. + */ +public class BackupProperties { + + private double indexSizeMB; + private int indexFileCount; + + private Properties properties; + + private BackupProperties(Properties properties) { + this.properties = properties; + } + + public static BackupProperties create(String backupName, + String collectionName, + String extCollectionName, + String configName) { + Properties properties = new Properties(); + properties.put(BackupManager.BACKUP_NAME_PROP, backupName); + properties.put(BackupManager.COLLECTION_NAME_PROP, collectionName); + properties.put(BackupManager.COLLECTION_ALIAS_PROP, extCollectionName); + properties.put(CollectionAdminParams.COLL_CONF, configName); + properties.put(BackupManager.START_TIME_PROP, Instant.now().toString()); + properties.put(BackupManager.INDEX_VERSION_PROP, Version.LATEST.toString()); + + return new BackupProperties(properties); + } + + public static Optional readFromLatest(BackupRepository repository, URI backupPath) throws IOException { + Optional lastBackupId = BackupId.findMostRecent(repository.listAllOrEmpty(backupPath)); + + if (!lastBackupId.isPresent()) { + return Optional.empty(); + } + + return Optional.of(readFrom(repository, backupPath, lastBackupId.get().getBackupPropsName())); + } + + public static BackupProperties readFrom(BackupRepository repository, URI backupPath, String fileName) throws IOException { + Properties props = new Properties(); + try (Reader is = new InputStreamReader(new PropertiesInputStream( + repository.openInput(backupPath, fileName, IOContext.DEFAULT)), StandardCharsets.UTF_8)) { + props.load(is); + return new BackupProperties(props); + } + } + + public List getAllShardBackupIdFiles() { + return properties.entrySet() + .stream() + .filter(entry -> entry.getKey().toString().endsWith(".md")) + .map(entry -> entry.getValue().toString()) + .collect(Collectors.toList()); + } + + public void countIndexFiles(int numFiles, double sizeMB) { + indexSizeMB += sizeMB; + indexFileCount += numFiles; + } + + public Optional getShardBackupIdFor(String shardName) { + String key = getKeyForShardBackupId(shardName); + if (properties.containsKey(key)) { + return Optional.of(properties.getProperty(key)); + } + return Optional.empty(); + } + + public String putAndGetShardBackupIdFor(String shardName, int backupId) { + String shardBackupId = String.format(Locale.ROOT, "md_%s_id_%d", shardName, backupId); + properties.put(getKeyForShardBackupId(shardName), shardBackupId); + return shardBackupId; + } + + public static Optional backupIdOfShardBackupId(String shardBackupIdName) { + if (!shardBackupIdName.startsWith("md")) { + return Optional.empty(); + } + try { + int id = Integer.parseInt(shardBackupIdName.substring(shardBackupIdName.lastIndexOf("_") + 1)); + return Optional.of(new BackupId(id)); + } catch (NumberFormatException | IndexOutOfBoundsException e) { + return Optional.empty(); + } + } + + private String getKeyForShardBackupId(String shardName) { + return shardName+".md"; + } + + + public void store(Writer propsWriter) throws IOException { + properties.put("indexSizeMB", String.valueOf(indexSizeMB)); + properties.put("indexFileCount", String.valueOf(indexFileCount)); + properties.store(propsWriter, "Backup properties file"); + } + + public String getCollection() { + return properties.getProperty(BackupManager.COLLECTION_NAME_PROP); + } + + + public String getCollectionAlias() { + return properties.getProperty(BackupManager.COLLECTION_ALIAS_PROP); + } + + public String getConfigName() { + return properties.getProperty(CollectionAdminParams.COLL_CONF); + } + + public String getStartTime() { + return properties.getProperty(BackupManager.START_TIME_PROP); + } + + public String getIndexVersion() { + return properties.getProperty(BackupManager.INDEX_VERSION_PROP); + } + + public Map getDetails() { + Map result = new HashMap<>(properties); + result.remove(BackupManager.BACKUP_NAME_PROP); + result.remove(BackupManager.COLLECTION_NAME_PROP); + result.put("indexSizeMB", Double.valueOf(properties.getProperty("indexSizeMB"))); + result.put("indexFileCount", Integer.valueOf(properties.getProperty("indexFileCount"))); + + Map shardBackupIds = new HashMap<>(); + Iterator keyIt = result.keySet().iterator(); + while (keyIt.hasNext()) { + String key = keyIt.next().toString(); + if (key.endsWith(".md")) { + shardBackupIds.put(key.substring(0, key.length() - 3), properties.getProperty(key)); + keyIt.remove(); + } + } + result.put("shardBackupIds", shardBackupIds); + return result; + } + + public String getBackupName() { + return properties.getProperty(BackupManager.BACKUP_NAME_PROP); + } +} diff --git a/solr/core/src/java/org/apache/solr/core/backup/Checksum.java b/solr/core/src/java/org/apache/solr/core/backup/Checksum.java new file mode 100644 index 00000000000..8c3e62f249b --- /dev/null +++ b/solr/core/src/java/org/apache/solr/core/backup/Checksum.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.core.backup; + +import java.util.Objects; + +public class Checksum { + public final long checksum; + public final long size; + + public Checksum(long checksum, long size) { + this.checksum = checksum; + this.size = size; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Checksum checksum = (Checksum) o; + return size == checksum.size && + this.checksum == checksum.checksum; + } + + @Override + public int hashCode() { + return Objects.hash(checksum, size); + } + +} diff --git a/solr/core/src/java/org/apache/solr/core/backup/ShardBackupId.java b/solr/core/src/java/org/apache/solr/core/backup/ShardBackupId.java new file mode 100644 index 00000000000..6118a336dfb --- /dev/null +++ b/solr/core/src/java/org/apache/solr/core/backup/ShardBackupId.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.core.backup; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.solr.common.util.Utils; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.apache.solr.util.PropertiesInputStream; + +/** + * ShardBackupId is a metadata (json) file storing all the index files needed for a specific backed up commit. + * To avoid file names duplication between multiple backups and shards, + * each index file is stored with an uniqueName and the mapping from uniqueName to its original name is also stored here + */ +public class ShardBackupId { + private Map allFiles = new HashMap<>(); + private List uniqueFileNames = new ArrayList<>(); + + public void addBackedFile(String uniqueFileName, String originalFileName, Checksum fileChecksum) { + addBackedFile(new BackedFile(uniqueFileName, originalFileName, fileChecksum)); + } + + public int numFiles() { + return uniqueFileNames.size(); + } + + public long totalSize() { + return allFiles.values().stream().map(bf -> bf.fileChecksum.size).reduce(0L, Long::sum); + } + + public void addBackedFile(BackedFile backedFile) { + allFiles.put(backedFile.originalFileName, backedFile); + uniqueFileNames.add(backedFile.uniqueFileName); + } + + public Optional getFile(String originalFileName) { + return Optional.ofNullable(allFiles.get(originalFileName)); + } + + public List listUniqueFileNames() { + return Collections.unmodifiableList(uniqueFileNames); + } + + public static ShardBackupId empty() { + return new ShardBackupId(); + } + + public static ShardBackupId from(BackupRepository repository, URI dir, String filename) throws IOException { + if (!repository.exists(repository.resolve(dir, filename))) { + return null; + } + + try (IndexInput is = repository.openInput(dir, filename, IOContext.DEFAULT)) { + return from(new PropertiesInputStream(is)); + } + } + + /** + * Storing ShardBackupId at {@code folderURI} with name {@code filename}. + * If a file already existed there, overwrite it. + */ + public void store(BackupRepository repository, URI folderURI, String filename) throws IOException { + URI fileURI = repository.resolve(folderURI, filename); + if (repository.exists(fileURI)) { + repository.delete(folderURI, Collections.singleton(filename), true); + } + + try (OutputStream os = repository.createOutput(repository.resolve(folderURI, filename))) { + store(os); + } + } + + public Collection listOriginalFileNames() { + return Collections.unmodifiableSet(allFiles.keySet()); + } + + private void store(OutputStream os) throws IOException { + @SuppressWarnings({"rawtypes"}) + Map map = new HashMap<>(); + + for (BackedFile backedFile : allFiles.values()) { + Map fileMap = new HashMap<>(); + fileMap.put("fileName", backedFile.originalFileName); + fileMap.put("checksum", backedFile.fileChecksum.checksum); + fileMap.put("size", backedFile.fileChecksum.size); + map.put(backedFile.uniqueFileName, fileMap); + } + + Utils.writeJson(map, os, false); + } + + private static ShardBackupId from(InputStream is) { + @SuppressWarnings({"unchecked"}) + Map map = (Map) Utils.fromJSON(is); + ShardBackupId shardBackupId = new ShardBackupId(); + for (String uniqueFileName : map.keySet()) { + @SuppressWarnings({"unchecked"}) + Map fileMap = (Map) map.get(uniqueFileName); + + String fileName = (String) fileMap.get("fileName"); + long checksum = (long) fileMap.get("checksum"); + long size = (long) fileMap.get("size"); + shardBackupId.addBackedFile(new BackedFile(uniqueFileName, fileName, new Checksum(checksum, size))); + } + + return shardBackupId; + } + + public static class BackedFile { + public final String uniqueFileName; + public final String originalFileName; + public final Checksum fileChecksum; + + BackedFile(String uniqueFileName, String originalFileName, Checksum fileChecksum) { + this.uniqueFileName = uniqueFileName; + this.originalFileName = originalFileName; + this.fileChecksum = fileChecksum; + } + } +} diff --git a/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepository.java b/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepository.java index d750f590039..e173ed293ee 100644 --- a/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepository.java +++ b/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepository.java @@ -20,11 +20,20 @@ import java.io.IOException; import java.io.OutputStream; import java.net.URI; +import java.util.Collection; +import java.util.Optional; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.IOUtils; import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.core.DirectoryFactory; +import org.apache.solr.core.backup.Checksum; import org.apache.solr.util.plugin.NamedListInitializedPlugin; /** @@ -47,8 +56,7 @@ enum PathType { * Otherwise return the default configuration value for the {@linkplain CoreAdminParams#BACKUP_LOCATION} parameter. */ default String getBackupLocation(String override) { - // If override is null and default backup location is unset, what do we do? - return override != null ? override : getConfigProperty(CoreAdminParams.BACKUP_LOCATION); + return Optional.ofNullable(override).orElse(getConfigProperty(CoreAdminParams.BACKUP_LOCATION)); } /** @@ -63,7 +71,7 @@ default String getBackupLocation(String override) { * @param path The path specified by the user. * @return the URI representation of the user supplied value */ - URI createURI(String path); + URI createURI(String path); /** * This method resolves a URI using the specified path components (as method arguments). @@ -136,6 +144,7 @@ default String getBackupLocation(String override) { /** * This method creates a directory at the specified path. + * If the directory already exist, this will be a no-op. * * @param path * The path where the directory needs to be created. @@ -166,7 +175,9 @@ default String getBackupLocation(String override) { * @throws IOException * in case of errors */ - void copyFileFrom(Directory sourceDir, String fileName, URI dest) throws IOException; + default void copyFileFrom(Directory sourceDir, String fileName, URI dest) throws IOException { + copyIndexFileFrom(sourceDir, fileName, dest, fileName); + } /** * Copy a file from specified sourceRepo to the destination directory (i.e. restore). @@ -180,5 +191,89 @@ default String getBackupLocation(String override) { * @throws IOException * in case of errors. */ - void copyFileTo(URI sourceRepo, String fileName, Directory dest) throws IOException; + default void copyFileTo(URI sourceRepo, String fileName, Directory dest) throws IOException { + copyIndexFileTo(sourceRepo, fileName, dest, fileName); + } + + /** + * List all files or directories directly under {@code path}. + * @return an empty array in case of IOException + */ + default String[] listAllOrEmpty(URI path) { + try { + return this.listAll(path); + } catch (IOException e) { + return new String[0]; + } + } + + default void copyIndexFileFrom(Directory sourceDir, String sourceFileName, Directory destDir, String destFileName) throws IOException { + boolean success = false; + try (ChecksumIndexInput is = sourceDir.openChecksumInput(sourceFileName, DirectoryFactory.IOCONTEXT_NO_CACHE); + IndexOutput os = destDir.createOutput(destFileName, DirectoryFactory.IOCONTEXT_NO_CACHE)) { + os.copyBytes(is, is.length() - CodecUtil.footerLength()); + + // ensure that index file is not corrupted + CodecUtil.checkFooter(is); + CodecUtil.writeFooter(os); + success = true; + } finally { + if (!success) { + IOUtils.deleteFilesIgnoringExceptions(destDir, destFileName); + } + } + } + + /** + * Delete {@code files} at {@code path} + * @since 8.3.0 + */ + default void delete(URI path, Collection files, boolean ignoreNoSuchFileException) throws IOException { + throw new UnsupportedOperationException(); + } + + /** + * Get checksum of {@code fileName} at {@code dir}. + * This method only be called on Lucene index files + * @since 8.3.0 + */ + default Checksum checksum(Directory dir, String fileName) throws IOException { + try (IndexInput in = dir.openChecksumInput(fileName, IOContext.READONCE)) { + return new Checksum(CodecUtil.retrieveChecksum(in), in.length()); + } + } + + /** + * Copy an index file from specified sourceDir to the destination repository (i.e. backup). + * + * @param sourceDir + * The source directory hosting the file to be copied. + * @param sourceFileName + * The name of the file to by copied + * @param destDir + * The destination backup location. + * @throws IOException + * in case of errors + * @throws CorruptIndexException + * in case checksum of the file does not match with precomputed checksum stored at the end of the file + * @since 8.3.0 + */ + default void copyIndexFileFrom(Directory sourceDir, String sourceFileName, URI destDir, String destFileName) throws IOException { + throw new UnsupportedOperationException(); + } + + /** + * Copy an index file from specified sourceRepo to the destination directory (i.e. restore). + * + * @param sourceRepo + * The source URI hosting the file to be copied. + * @param dest + * The destination where the file should be copied. + * @throws IOException + * in case of errors. + * @since 8.3.0 + */ + default void copyIndexFileTo(URI sourceRepo, String sourceFileName, Directory dest, String destFileName) throws IOException { + throw new UnsupportedOperationException(); + } } diff --git a/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java b/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java index 9e02b21c109..6d57a90e856 100644 --- a/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java @@ -60,18 +60,24 @@ public BackupRepositoryFactory(PluginInfo[] backupRepoPlugins) { if (this.defaultBackupRepoPlugin != null) { log.info("Default configuration for backup repository is with configuration params {}", - defaultBackupRepoPlugin); + defaultBackupRepoPlugin); } } } + @SuppressWarnings({"unchecked"}) public BackupRepository newInstance(SolrResourceLoader loader, String name) { Objects.requireNonNull(loader); Objects.requireNonNull(name); PluginInfo repo = Objects.requireNonNull(backupRepoPluginByName.get(name), - "Could not find a backup repository with name " + name); + "Could not find a backup repository with name " + name); BackupRepository result = loader.newInstance(repo.className, BackupRepository.class); + if ("trackingBackupRepository".equals(name) && repo.initArgs.get("factory") == null) { + repo.initArgs.add("factory", this); + repo.initArgs.add("loader", loader); + } + result.init(repo.initArgs); return result; } diff --git a/solr/core/src/java/org/apache/solr/core/backup/repository/HdfsBackupRepository.java b/solr/core/src/java/org/apache/solr/core/backup/repository/HdfsBackupRepository.java index ada0b57a29f..b1f1c5e4bbb 100644 --- a/solr/core/src/java/org/apache/solr/core/backup/repository/HdfsBackupRepository.java +++ b/solr/core/src/java/org/apache/solr/core/backup/repository/HdfsBackupRepository.java @@ -21,6 +21,8 @@ import java.io.OutputStream; import java.net.URI; import java.net.URISyntaxException; +import java.nio.file.NoSuchFileException; +import java.util.Collection; import java.util.Objects; import java.lang.invoke.MethodHandles; @@ -194,18 +196,33 @@ public void deleteDirectory(URI path) throws IOException { } @Override - public void copyFileFrom(Directory sourceDir, String fileName, URI dest) throws IOException { - try (HdfsDirectory dir = new HdfsDirectory(new Path(dest), NoLockFactory.INSTANCE, - hdfsConfig, copyBufferSize)) { - dir.copyFrom(sourceDir, fileName, fileName, DirectoryFactory.IOCONTEXT_NO_CACHE); + public void copyIndexFileFrom(Directory sourceDir, String sourceFileName, URI destDir, String destFileName) throws IOException { + try (HdfsDirectory dir = new HdfsDirectory(new Path(destDir), NoLockFactory.INSTANCE, + hdfsConfig, copyBufferSize)) { + copyIndexFileFrom(sourceDir, sourceFileName, dir, destFileName); } } @Override - public void copyFileTo(URI sourceRepo, String fileName, Directory dest) throws IOException { + public void copyIndexFileTo(URI sourceRepo, String sourceFileName, Directory dest, String destFileName) throws IOException { try (HdfsDirectory dir = new HdfsDirectory(new Path(sourceRepo), NoLockFactory.INSTANCE, - hdfsConfig, copyBufferSize)) { - dest.copyFrom(dir, fileName, fileName, DirectoryFactory.IOCONTEXT_NO_CACHE); + hdfsConfig, copyBufferSize)) { + dest.copyFrom(dir, sourceFileName, destFileName, DirectoryFactory.IOCONTEXT_NO_CACHE); } } + + @Override + public void delete(URI path, Collection files, boolean ignoreNoSuchFileException) throws IOException { + if (files.isEmpty()) + return; + + for (String file : files) { + Path filePath = new Path(new Path(path), file); + boolean success = fileSystem.delete(filePath, false); + if (!ignoreNoSuchFileException && !success) { + throw new NoSuchFileException(filePath.toString()); + } + } + } + } diff --git a/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java b/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java index 612a61fd4bf..f16db6d693f 100644 --- a/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java +++ b/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java @@ -23,10 +23,13 @@ import java.net.URISyntaxException; import java.nio.file.FileVisitResult; import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import java.util.Collection; import java.util.Objects; import org.apache.lucene.store.Directory; @@ -83,7 +86,13 @@ public URI resolve(URI baseUri, String... pathComponents) { Path result = Paths.get(baseUri); for (int i = 0; i < pathComponents.length; i++) { - result = result.resolve(pathComponents[i]); + try { + result = result.resolve(pathComponents[i]); + } catch (Exception e) { + // unlikely to happen + throw new RuntimeException(e); + } + } return result.toUri(); @@ -91,7 +100,10 @@ public URI resolve(URI baseUri, String... pathComponents) { @Override public void createDirectory(URI path) throws IOException { - Files.createDirectory(Paths.get(path)); + Path p = Paths.get(path); + if (!Files.exists(p, LinkOption.NOFOLLOW_LINKS)) { + Files.createDirectory(p); + } } @Override @@ -130,6 +142,12 @@ public OutputStream createOutput(URI path) throws IOException { @Override public String[] listAll(URI dirPath) throws IOException { + // It is better to check the existence of the directory first since + // creating a FSDirectory will create a corresponds folder if the directory does not exist + if (!exists(dirPath)) { + return new String[0]; + } + try (FSDirectory dir = new NIOFSDirectory(Paths.get(dirPath), NoLockFactory.INSTANCE)) { return dir.listAll(); } @@ -141,16 +159,33 @@ public PathType getPathType(URI path) throws IOException { } @Override - public void copyFileFrom(Directory sourceDir, String fileName, URI dest) throws IOException { - try (FSDirectory dir = new NIOFSDirectory(Paths.get(dest), NoLockFactory.INSTANCE)) { - dir.copyFrom(sourceDir, fileName, fileName, DirectoryFactory.IOCONTEXT_NO_CACHE); + public void copyIndexFileFrom(Directory sourceDir, String sourceFileName, URI destDir, String destFileName) throws IOException { + try (FSDirectory dir = new NIOFSDirectory(Paths.get(destDir), NoLockFactory.INSTANCE)) { + copyIndexFileFrom(sourceDir, sourceFileName, dir, destFileName); } } @Override - public void copyFileTo(URI sourceDir, String fileName, Directory dest) throws IOException { + public void copyIndexFileTo(URI sourceDir, String sourceFileName, Directory dest, String destFileName) throws IOException { try (FSDirectory dir = new NIOFSDirectory(Paths.get(sourceDir), NoLockFactory.INSTANCE)) { - dest.copyFrom(dir, fileName, fileName, DirectoryFactory.IOCONTEXT_NO_CACHE); + dest.copyFrom(dir, sourceFileName, destFileName, DirectoryFactory.IOCONTEXT_NO_CACHE); + } + } + + @Override + public void delete(URI path, Collection files, boolean ignoreNoSuchFileException) throws IOException { + if (files.isEmpty()) + return; + + try (FSDirectory dir = new NIOFSDirectory(Paths.get(path), NoLockFactory.INSTANCE)) { + for (String file : files) { + try { + dir.deleteFile(file); + } catch (NoSuchFileException e) { + if (!ignoreNoSuchFileException) + throw e; + } + } } } diff --git a/solr/core/src/java/org/apache/solr/handler/IncrementalBackupPaths.java b/solr/core/src/java/org/apache/solr/handler/IncrementalBackupPaths.java new file mode 100644 index 00000000000..2ec14c488d7 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/IncrementalBackupPaths.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler; + +import java.io.IOException; +import java.net.URI; + +import org.apache.solr.core.backup.repository.BackupRepository; + +/** + * Utility class for getting paths related to incremental backup + */ +public class IncrementalBackupPaths { + + private BackupRepository repository; + private URI backupLoc; + + public IncrementalBackupPaths(BackupRepository repository, URI backupLoc) { + this.repository = repository; + this.backupLoc = backupLoc; + } + + public URI getIndexDir() { + return repository.resolve(backupLoc, "index"); + } + + public URI getShardBackupIdDir() { + return repository.resolve(backupLoc, "shard_backup_ids"); + } + + public URI getBackupLocation() { + return backupLoc; + } + + public void createFolders() throws IOException { + if (!repository.exists(backupLoc)) { + repository.createDirectory(backupLoc); + } + URI indexDir = getIndexDir(); + if (!repository.exists(indexDir)) { + repository.createDirectory(indexDir); + } + + URI shardBackupIdDir = getShardBackupIdDir(); + if (!repository.exists(shardBackupIdDir)) { + repository.createDirectory(shardBackupIdDir); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/IncrementalShardBackup.java b/solr/core/src/java/org/apache/solr/handler/IncrementalShardBackup.java new file mode 100644 index 00000000000..a4d218569fa --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/IncrementalShardBackup.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.net.URI; +import java.time.Instant; +import java.util.Collection; +import java.util.Optional; +import java.util.UUID; + +import org.apache.commons.math3.util.Precision; +import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.store.Directory; +import org.apache.solr.cloud.CloudDescriptor; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.DirectoryFactory; +import org.apache.solr.core.IndexDeletionPolicyWrapper; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.backup.Checksum; +import org.apache.solr.core.backup.ShardBackupId; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Backup a core in an incremental way by leveraging information from previous backups ({@link ShardBackupId} + */ +public class IncrementalShardBackup { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private SolrCore solrCore; + + private IncrementalBackupPaths incBackupFiles; + private BackupRepository backupRepo; + + private String prevShardBackupIdFile; + private String shardBackupIdFile; + + /** + * + * @param prevShardBackupIdFile previous ShardBackupId file which will be used for skipping + * uploading index files already present in this file. + * @param shardBackupIdFile file where all meta data of this backup will be stored to. + */ + public IncrementalShardBackup(BackupRepository backupRepo, SolrCore solrCore, IncrementalBackupPaths incBackupFiles, + String prevShardBackupIdFile, String shardBackupIdFile) { + this.backupRepo = backupRepo; + this.solrCore = solrCore; + this.incBackupFiles = incBackupFiles; + this.prevShardBackupIdFile = prevShardBackupIdFile; + this.shardBackupIdFile = shardBackupIdFile; + } + + @SuppressWarnings({"rawtypes"}) + public NamedList backup() throws Exception { + final IndexCommit indexCommit = getAndSaveIndexCommit(); + try { + return backup(indexCommit); + } finally { + solrCore.getDeletionPolicy().releaseCommitPoint(indexCommit.getGeneration()); + } + } + + /** + * Returns {@link IndexDeletionPolicyWrapper#getAndSaveLatestCommit}. + *

+ * Note: + *

    + *
  • This method does error handling when the commit can't be found and wraps them in {@link SolrException} + *
    • + *
  • If this method returns, the result will be non null, and the caller MUST + * call {@link IndexDeletionPolicyWrapper#releaseCommitPoint} when finished + *
    • + *
+ */ + private IndexCommit getAndSaveIndexCommit() throws IOException { + final IndexDeletionPolicyWrapper delPolicy = solrCore.getDeletionPolicy(); + final IndexCommit commit = delPolicy.getAndSaveLatestCommit(); + if (null == commit) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Index does not yet have any commits for core " + + solrCore.getName()); + } + if (log.isDebugEnabled()) { + log.debug("Using latest commit: generation={}", commit.getGeneration()); + } + return commit; + } + + // note: remember to reserve the indexCommit first so it won't get deleted concurrently + @SuppressWarnings({"rawtypes"}) + protected NamedList backup(final IndexCommit indexCommit) throws Exception { + assert indexCommit != null; + URI backupLocation = incBackupFiles.getBackupLocation(); + log.info("Creating backup snapshot at {} shardBackupIdFile:{}", backupLocation, shardBackupIdFile); + NamedList details = new NamedList<>(); + details.add("startTime", Instant.now().toString()); + + Collection files = indexCommit.getFileNames(); + Directory dir = solrCore.getDirectoryFactory().get(solrCore.getIndexDir(), + DirectoryFactory.DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType); + try { + BackupStats stats = incrementalCopy(files, dir); + details.add("indexFileCount", stats.fileCount); + details.add("uploadedIndexFileCount", stats.uploadedFileCount); + details.add("indexSizeMB", stats.getIndexSizeMB()); + details.add("uploadedIndexFileMB", stats.getTotalUploadedMB()); + } finally { + solrCore.getDirectoryFactory().release(dir); + } + + CloudDescriptor cd = solrCore.getCoreDescriptor().getCloudDescriptor(); + if (cd != null) { + details.add("shard", cd.getShardId()); + } + + details.add("endTime", Instant.now().toString()); + details.add("shardBackupId", shardBackupIdFile); + log.info("Done creating backup snapshot at {} shardBackupIdFile:{}", backupLocation, shardBackupIdFile); + return details; + } + + private ShardBackupId getPrevBackupPoint() throws IOException { + if (prevShardBackupIdFile == null) { + return ShardBackupId.empty(); + } + return ShardBackupId.from(backupRepo, incBackupFiles.getShardBackupIdDir(), prevShardBackupIdFile); + } + + private BackupStats incrementalCopy(Collection indexFiles, Directory dir) throws IOException { + ShardBackupId oldBackupPoint = getPrevBackupPoint(); + ShardBackupId currentBackupPoint = ShardBackupId.empty(); + URI indexDir = incBackupFiles.getIndexDir(); + BackupStats backupStats = new BackupStats(); + + for(String fileName : indexFiles) { + Optional opBackedFile = oldBackupPoint.getFile(fileName); + Checksum originalFileCS = backupRepo.checksum(dir, fileName); + + if (opBackedFile.isPresent()) { + ShardBackupId.BackedFile backedFile = opBackedFile.get(); + Checksum existedFileCS = backedFile.fileChecksum; + if (existedFileCS.equals(originalFileCS)) { + currentBackupPoint.addBackedFile(opBackedFile.get()); + backupStats.skippedUploadingFile(existedFileCS); + continue; + } + } + + String backedFileName = UUID.randomUUID().toString(); + backupRepo.copyIndexFileFrom(dir, fileName, indexDir, backedFileName); + + currentBackupPoint.addBackedFile(backedFileName, fileName, originalFileCS); + backupStats.uploadedFile(originalFileCS); + } + + currentBackupPoint.store(backupRepo, incBackupFiles.getShardBackupIdDir(), shardBackupIdFile); + return backupStats; + } + + private static class BackupStats { + private int fileCount; + private int uploadedFileCount; + private long indexSize; + private long totalUploadedBytes; + + public void uploadedFile(Checksum file) { + fileCount++; + uploadedFileCount++; + indexSize += file.size; + totalUploadedBytes += file.size; + } + + public void skippedUploadingFile(Checksum existedFile) { + fileCount++; + indexSize += existedFile.size; + } + + public double getIndexSizeMB() { + return Precision.round(indexSize / (1024.0 * 1024), 3); + } + + public double getTotalUploadedMB() { + return Precision.round(totalUploadedBytes / (1024.0 * 1024), 3); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java index 9908292d91b..71ebd4952c4 100644 --- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java @@ -523,7 +523,7 @@ private void restore(SolrParams params, SolrQueryResponse rsp, SolrQueryRequest name = "snapshot." + name; } - RestoreCore restoreCore = new RestoreCore(repo, core, locationUri, name); + RestoreCore restoreCore = RestoreCore.create(repo, core, locationUri, name); try { MDC.put("RestoreCore.core", core.getName()); MDC.put("RestoreCore.backupLocation", location); diff --git a/solr/core/src/java/org/apache/solr/handler/RestoreCore.java b/solr/core/src/java/org/apache/solr/handler/RestoreCore.java index 3e12d4b101c..30ac07a9ff8 100644 --- a/solr/core/src/java/org/apache/solr/handler/RestoreCore.java +++ b/solr/core/src/java/org/apache/solr/handler/RestoreCore.java @@ -16,11 +16,16 @@ */ package org.apache.solr.handler; +import java.io.IOException; import java.lang.invoke.MethodHandles; import java.net.URI; import java.text.SimpleDateFormat; +import java.util.Arrays; import java.util.Date; +import java.util.HashSet; import java.util.Locale; +import java.util.Optional; +import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.Future; @@ -31,6 +36,8 @@ import org.apache.solr.common.SolrException; import org.apache.solr.core.DirectoryFactory; import org.apache.solr.core.SolrCore; +import org.apache.solr.core.backup.Checksum; +import org.apache.solr.core.backup.ShardBackupId; import org.apache.solr.core.backup.repository.BackupRepository; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,16 +46,25 @@ public class RestoreCore implements Callable { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private final String backupName; - private final URI backupLocation; private final SolrCore core; - private final BackupRepository backupRepo; + private RestoreRepository repository; - public RestoreCore(BackupRepository backupRepo, SolrCore core, URI location, String name) { - this.backupRepo = backupRepo; + private RestoreCore(SolrCore core, RestoreRepository repository) { this.core = core; - this.backupLocation = location; - this.backupName = name; + this.repository = repository; + } + + public static RestoreCore create(BackupRepository backupRepo, SolrCore core, URI location, String backupname) { + RestoreRepository repository = new BasicRestoreRepository(backupRepo.resolve(location, backupname), backupRepo); + return new RestoreCore(core, repository); + } + + public static RestoreCore createWithMetaFile(BackupRepository repo, SolrCore core, URI location, String metaFile) throws IOException { + IncrementalBackupPaths incBackupFiles = new IncrementalBackupPaths(repo, location); + URI shardBackupIdDir = incBackupFiles.getShardBackupIdDir(); + ShardBackupIdRestoreRepository resolver = new ShardBackupIdRestoreRepository(location, incBackupFiles.getIndexDir(), + repo, ShardBackupId.from(repo, shardBackupIdDir, metaFile)); + return new RestoreCore(core, resolver); } @Override @@ -57,8 +73,6 @@ public Boolean call() throws Exception { } public boolean doRestore() throws Exception { - - URI backupPath = backupRepo.resolve(backupLocation, backupName); SimpleDateFormat dateFormat = new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT); String restoreIndexName = "restore." + dateFormat.format(new Date()); String restoreIndexPath = core.getDataDir() + restoreIndexName; @@ -69,31 +83,34 @@ public boolean doRestore() throws Exception { try { restoreIndexDir = core.getDirectoryFactory().get(restoreIndexPath, - DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType); + DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType); //Prefer local copy. indexDir = core.getDirectoryFactory().get(indexDirPath, - DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType); - + DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType); + Set indexDirFiles = new HashSet<>(Arrays.asList(indexDir.listAll())); //Move all files from backupDir to restoreIndexDir - for (String filename : backupRepo.listAll(backupPath)) { + for (String filename : repository.listAllFiles()) { checkInterrupted(); - log.info("Copying file {} to restore directory ", filename); - try (IndexInput indexInput = backupRepo.openInput(backupPath, filename, IOContext.READONCE)) { - Long checksum = null; - try { - checksum = CodecUtil.retrieveChecksum(indexInput); - } catch (Exception e) { - log.warn("Could not read checksum from index file: {}", filename, e); - } - long length = indexInput.length(); - IndexFetcher.CompareResult compareResult = IndexFetcher.compareFile(indexDir, filename, length, checksum); - if (!compareResult.equal || - (IndexFetcher.filesToAlwaysDownloadIfNoChecksums(filename, length, compareResult))) { - backupRepo.copyFileTo(backupPath, filename, restoreIndexDir); + try { + if (indexDirFiles.contains(filename)) { + Checksum cs = repository.checksum(filename); + IndexFetcher.CompareResult compareResult; + if (cs == null) { + compareResult = new IndexFetcher.CompareResult(); + compareResult.equal = false; + } else { + compareResult = IndexFetcher.compareFile(indexDir, filename, cs.size, cs.checksum); + } + if (!compareResult.equal || + (IndexFetcher.filesToAlwaysDownloadIfNoChecksums(filename, cs.size, compareResult))) { + repository.repoCopy(filename, restoreIndexDir); + } else { + //prefer local copy + repository.localCopy(indexDir, filename, restoreIndexDir); + } } else { - //prefer local copy - restoreIndexDir.copyFrom(indexDir, filename, filename, IOContext.READONCE); + repository.repoCopy(filename, restoreIndexDir); } } catch (Exception e) { log.warn("Exception while restoring the backup index ", e); @@ -115,7 +132,7 @@ public boolean doRestore() throws Exception { Directory dir = null; try { dir = core.getDirectoryFactory().get(core.getDataDir(), DirectoryFactory.DirContext.META_DATA, - core.getSolrConfig().indexConfig.lockType); + core.getSolrConfig().indexConfig.lockType); dir.deleteFile(IndexFetcher.INDEX_PROPERTIES); } finally { if (dir != null) { @@ -160,4 +177,106 @@ private void openNewSearcher() throws Exception { waitSearcher[0].get(); } } + + /** + * A minimal version of {@link BackupRepository} used for restoring + */ + private interface RestoreRepository { + String[] listAllFiles() throws IOException; + IndexInput openInput(String filename) throws IOException; + void repoCopy(String filename, Directory dest) throws IOException; + void localCopy(Directory src, String filename, Directory dest) throws IOException; + Checksum checksum(String filename) throws IOException; + } + + /** + * A basic {@link RestoreRepository} simply delegates all calls to {@link BackupRepository} + */ + private static class BasicRestoreRepository implements RestoreRepository { + protected final URI backupPath; + protected final BackupRepository repository; + + public BasicRestoreRepository(URI backupPath, BackupRepository repository) { + this.backupPath = backupPath; + this.repository = repository; + } + + public String[] listAllFiles() throws IOException { + return repository.listAll(backupPath); + } + + public IndexInput openInput(String filename) throws IOException { + return repository.openInput(backupPath, filename, IOContext.READONCE); + } + + public void repoCopy(String filename, Directory dest) throws IOException { + repository.copyFileTo(backupPath, filename, dest); + } + + public void localCopy(Directory src, String filename, Directory dest) throws IOException { + dest.copyFrom(src, filename, filename, IOContext.READONCE); + } + + public Checksum checksum(String filename) throws IOException { + try (IndexInput indexInput = repository.openInput(backupPath, filename, IOContext.READONCE)) { + try { + long checksum = CodecUtil.retrieveChecksum(indexInput); + long length = indexInput.length(); + return new Checksum(checksum, length); + } catch (Exception e) { + log.warn("Could not read checksum from index file: {}", filename, e); + } + } + return null; + } + } + + /** + * A {@link RestoreRepository} based on information stored in {@link ShardBackupId} + */ + private static class ShardBackupIdRestoreRepository implements RestoreRepository { + + private final ShardBackupId shardBackupId; + private final URI indexURI; + protected final URI backupPath; + protected final BackupRepository repository; + + public ShardBackupIdRestoreRepository(URI backupPath, URI indexURI, BackupRepository repository, ShardBackupId shardBackupId) { + this.shardBackupId = shardBackupId; + this.indexURI = indexURI; + this.backupPath = backupPath; + this.repository = repository; + } + + @Override + public String[] listAllFiles() { + return shardBackupId.listOriginalFileNames().toArray(new String[0]); + } + + @Override + public IndexInput openInput(String filename) throws IOException { + String storedFileName = getStoredFilename(filename); + return repository.openInput(indexURI, storedFileName, IOContext.READONCE); + } + + @Override + public void repoCopy(String filename, Directory dest) throws IOException { + String storedFileName = getStoredFilename(filename); + repository.copyIndexFileTo(this.indexURI, storedFileName, dest, filename); + } + + @Override + public void localCopy(Directory src, String filename, Directory dest) throws IOException { + dest.copyFrom(src, filename, filename, IOContext.READONCE); + } + + public Checksum checksum(String filename) { + Optional backedFile = shardBackupId.getFile(filename); + return backedFile.map(bf -> bf.fileChecksum).orElse(null); + } + + private String getStoredFilename(String filename) { + return shardBackupId.getFile(filename).get().uniqueFileName; + } + } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java b/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java index c295ad5d22d..0bd12940eea 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java @@ -22,13 +22,15 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; import org.apache.solr.core.backup.repository.BackupRepository; +import org.apache.solr.handler.IncrementalBackupPaths; +import org.apache.solr.handler.IncrementalShardBackup; import org.apache.solr.handler.SnapShooter; import static org.apache.solr.common.params.CommonParams.NAME; - class BackupCoreOp implements CoreAdminHandler.CoreAdminOp { @Override public void execute(CoreAdminHandler.CallInfo it) throws Exception { @@ -36,38 +38,48 @@ public void execute(CoreAdminHandler.CallInfo it) throws Exception { String cname = params.required().get(CoreAdminParams.CORE); String name = params.required().get(NAME); - + boolean incremental = params.getBool(CoreAdminParams.BACKUP_INCREMENTAL, false); + String backupMetadataFile = params.get(CoreAdminParams.SHARD_BACKUP_ID, null); + String prevBackupMetadataFile = params.get(CoreAdminParams.PREV_SHARD_BACKUP_ID, null); String repoName = params.get(CoreAdminParams.BACKUP_REPOSITORY); - BackupRepository repository = it.handler.coreContainer.newBackupRepository(repoName); - - String location = repository.getBackupLocation(params.get(CoreAdminParams.BACKUP_LOCATION)); - if (location == null) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'location' is not specified as a query" - + " parameter or as a default repository property"); - } - // An optional parameter to describe the snapshot to be backed-up. If this // parameter is not supplied, the latest index commit is backed-up. String commitName = params.get(CoreAdminParams.COMMIT_NAME); - URI locationUri = repository.createURI(location); - try (SolrCore core = it.handler.coreContainer.getCore(cname)) { - SnapShooter snapShooter = new SnapShooter(repository, core, locationUri, name, commitName); - // validateCreateSnapshot will create parent dirs instead of throw; that choice is dubious. - // But we want to throw. One reason is that - // this dir really should, in fact must, already exist here if triggered via a collection backup on a shared - // file system. Otherwise, perhaps the FS location isn't shared -- we want an error. - if (!snapShooter.getBackupRepository().exists(snapShooter.getLocation())) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "Directory to contain snapshots doesn't exist: " + snapShooter.getLocation() + ". " + - "Note that Backup/Restore of a SolrCloud collection " + - "requires a shared file system mounted at the same path on all nodes!"); + try (BackupRepository repository = it.handler.coreContainer.newBackupRepository(repoName); + SolrCore core = it.handler.coreContainer.getCore(cname)) { + String location = repository.getBackupLocation(params.get(CoreAdminParams.BACKUP_LOCATION)); + if (location == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'location' is not specified as a query" + + " parameter or as a default repository property"); + } + + URI locationUri = repository.createURI(location); + if (incremental) { + IncrementalBackupPaths incBackupFiles = new IncrementalBackupPaths(repository, locationUri); + IncrementalShardBackup incSnapShooter = new IncrementalShardBackup(repository, core, incBackupFiles, + prevBackupMetadataFile, backupMetadataFile); + @SuppressWarnings({"rawtypes"}) + NamedList rsp = incSnapShooter.backup(); + it.rsp.addResponse(rsp); + } else { + SnapShooter snapShooter = new SnapShooter(repository, core, locationUri, name, commitName); + // validateCreateSnapshot will create parent dirs instead of throw; that choice is dubious. + // But we want to throw. One reason is that + // this dir really should, in fact must, already exist here if triggered via a collection backup on a shared + // file system. Otherwise, perhaps the FS location isn't shared -- we want an error. + if (!snapShooter.getBackupRepository().exists(snapShooter.getLocation())) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Directory to contain snapshots doesn't exist: " + snapShooter.getLocation() + ". " + + "Note that Backup/Restore of a SolrCloud collection " + + "requires a shared file system mounted at the same path on all nodes!"); + } + snapShooter.validateCreateSnapshot(); + snapShooter.createSnapshot(); } - snapShooter.validateCreateSnapshot(); - snapShooter.createSnapshot(); } catch (Exception e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Failed to backup core=" + cname + " because " + e, e); + "Failed to backup core=" + cname + " because " + e, e); } } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 8d7e7cff34d..fc9bcdff5b1 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -19,17 +19,7 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; @@ -84,6 +74,9 @@ import org.apache.solr.common.util.Utils; import org.apache.solr.core.CloudConfig; import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.backup.BackupId; +import org.apache.solr.core.backup.BackupManager; +import org.apache.solr.core.backup.BackupProperties; import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.core.snapshots.CollectionSnapshotMetaData; import org.apache.solr.core.snapshots.SolrSnapshotManager; @@ -137,6 +130,7 @@ import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_VALUE; import static org.apache.solr.common.params.CollectionAdminParams.SKIP_NODE_ASSIGNMENT; import static org.apache.solr.common.params.CollectionParams.CollectionAction.*; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEBACKUP; import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.IN_PLACE_MOVE; import static org.apache.solr.common.params.CommonAdminParams.NUM_SUB_SHARDS; @@ -147,13 +141,8 @@ import static org.apache.solr.common.params.CommonParams.NAME; import static org.apache.solr.common.params.CommonParams.TIMING; import static org.apache.solr.common.params.CommonParams.VALUE_LONG; -import static org.apache.solr.common.params.CoreAdminParams.DATA_DIR; -import static org.apache.solr.common.params.CoreAdminParams.DELETE_DATA_DIR; -import static org.apache.solr.common.params.CoreAdminParams.DELETE_INDEX; -import static org.apache.solr.common.params.CoreAdminParams.DELETE_INSTANCE_DIR; -import static org.apache.solr.common.params.CoreAdminParams.DELETE_METRICS_HISTORY; -import static org.apache.solr.common.params.CoreAdminParams.INSTANCE_DIR; -import static org.apache.solr.common.params.CoreAdminParams.ULOG_DIR; +import static org.apache.solr.common.params.CoreAdminParams.*; +import static org.apache.solr.common.params.CoreAdminParams.PURGE_BACKUP; import static org.apache.solr.common.params.ShardParams._ROUTE_; import static org.apache.solr.common.util.StrUtils.formatString; @@ -1065,6 +1054,8 @@ public Map execute(SolrQueryRequest req, SolrQueryResponse rsp, } } + boolean incremental = req.getParams().getBool(CoreAdminParams.BACKUP_INCREMENTAL, false); + // Check if the specified location is valid for this repository. final URI uri = repository.createURI(location); try { @@ -1072,7 +1063,7 @@ public Map execute(SolrQueryRequest req, SolrQueryResponse rsp, throw new SolrException(ErrorCode.SERVER_ERROR, "specified location " + uri + " does not exist."); } } catch (IOException ex) { - throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to check the existance of " + uri + ". Is it valid?", ex); + throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to check the existence of " + uri + ". Is it valid?", ex); } String strategy = req.getParams().get(CollectionAdminParams.INDEX_BACKUP_STRATEGY, CollectionAdminParams.COPY_FILES_STRATEGY); @@ -1080,24 +1071,21 @@ public Map execute(SolrQueryRequest req, SolrQueryResponse rsp, throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown index backup strategy " + strategy); } - final Map params = copy(req.getParams(), null, NAME, COLLECTION_PROP, FOLLOW_ALIASES, CoreAdminParams.COMMIT_NAME); + Map params = copy(req.getParams(), null, NAME, COLLECTION_PROP, + FOLLOW_ALIASES, CoreAdminParams.COMMIT_NAME, CoreAdminParams.MAX_NUM_BACKUP); params.put(CoreAdminParams.BACKUP_LOCATION, location); if (repo != null) { params.put(CoreAdminParams.BACKUP_REPOSITORY, repo); } params.put(CollectionAdminParams.INDEX_BACKUP_STRATEGY, strategy); + params.put(CoreAdminParams.BACKUP_INCREMENTAL, incremental); return params; }), RESTORE_OP(RESTORE, (req, rsp, h) -> { req.getParams().required().check(NAME, COLLECTION_PROP); final String collectionName = SolrIdentifierValidator.validateCollectionName(req.getParams().get(COLLECTION_PROP)); - final ClusterState clusterState = h.coreContainer.getZkController().getClusterState(); - //We always want to restore into an collection name which doesn't exist yet. - if (clusterState.hasCollection(collectionName)) { - throw new SolrException(ErrorCode.BAD_REQUEST, "Collection '" + collectionName + "' exists, no action taken."); - } if (h.coreContainer.getZkController().getZkStateReader().getAliases().hasAlias(collectionName)) { throw new SolrException(ErrorCode.BAD_REQUEST, "Collection '" + collectionName + "' is an existing alias, no action taken."); } @@ -1146,10 +1134,94 @@ public Map execute(SolrQueryRequest req, SolrQueryResponse rsp, } // from CREATE_OP: copy(req.getParams(), params, COLL_CONF, REPLICATION_FACTOR, NRT_REPLICAS, TLOG_REPLICAS, - PULL_REPLICAS, CREATE_NODE_SET, CREATE_NODE_SET_SHUFFLE); + PULL_REPLICAS, CREATE_NODE_SET, CREATE_NODE_SET_SHUFFLE, BACKUP_ID); copyPropertiesWithPrefix(req.getParams(), params, COLL_PROP_PREFIX); return params; }), + DELETEBACKUP_OP(DELETEBACKUP, (req, rsp, h) -> { + req.getParams().required().check(NAME); + + CoreContainer cc = h.coreContainer; + String repo = req.getParams().get(CoreAdminParams.BACKUP_REPOSITORY); + try (BackupRepository repository = cc.newBackupRepository(repo)) { + + String location = repository.getBackupLocation(req.getParams().get(CoreAdminParams.BACKUP_LOCATION)); + if (location == null) { + //Refresh the cluster property file to make sure the value set for location is the latest + // Check if the location is specified in the cluster property. + location = new ClusterProperties(h.coreContainer.getZkController().getZkClient()).getClusterProperty("location", null); + if (location == null) { + throw new SolrException(ErrorCode.BAD_REQUEST, "'location' is not specified as a query" + + " parameter or as a default repository property or as a cluster property."); + } + } + + // Check if the specified location is valid for this repository. + URI uri = repository.createURI(location); + try { + if (!repository.exists(uri)) { + throw new SolrException(ErrorCode.BAD_REQUEST, "specified location " + uri + " does not exist."); + } + } catch (IOException ex) { + throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to check the existance of " + uri + ". Is it valid?", ex); + } + + if (req.getParams().get(MAX_NUM_BACKUP) == null && + req.getParams().get(PURGE_BACKUP) == null && + req.getParams().get(BACKUP_ID) == null) { + throw new SolrException(BAD_REQUEST, String.format(Locale.ROOT, "%s, %s or %s param must be provided", CoreAdminParams.BACKUP_ID, CoreAdminParams.MAX_NUM_BACKUP, + CoreAdminParams.PURGE_BACKUP)); + } + + return copy(req.getParams(), null, NAME, + COLLECTION_PROP, BACKUP_REPOSITORY, BACKUP_LOCATION, BACKUP_ID, MAX_NUM_BACKUP, PURGE_BACKUP); + } + }), + @SuppressWarnings({"unchecked", "rawtypes"}) + LISTBACKUP_OP(LISTBACKUP, (req, rsp, h) -> { + req.getParams().required().check(NAME); + + CoreContainer cc = h.coreContainer; + String repo = req.getParams().get(CoreAdminParams.BACKUP_REPOSITORY); + try (BackupRepository repository = cc.newBackupRepository(repo)) { + + String location = repository.getBackupLocation(req.getParams().get(CoreAdminParams.BACKUP_LOCATION)); + if (location == null) { + //Refresh the cluster property file to make sure the value set for location is the latest + // Check if the location is specified in the cluster property. + location = new ClusterProperties(h.coreContainer.getZkController().getZkClient()).getClusterProperty(CoreAdminParams.BACKUP_LOCATION, null); + if (location == null) { + throw new SolrException(ErrorCode.BAD_REQUEST, "'location' is not specified as a query" + + " parameter or as a default repository property or as a cluster property."); + } + } + + String backupName = req.getParams().get(NAME); + URI backupLocation = repository.resolve(repository.createURI(location), backupName); + String[] subFiles = repository.listAllOrEmpty(backupLocation); + List propsFiles = BackupId.findAll(subFiles); + + NamedList results = new NamedList<>(); + ArrayList backups = new ArrayList<>(); + String collectionName = null; + for (BackupId backupId: propsFiles) { + BackupProperties properties = BackupProperties.readFrom(repository, backupLocation, backupId.getBackupPropsName()); + if (collectionName == null) { + collectionName = properties.getCollection(); + results.add(BackupManager.COLLECTION_NAME_PROP, collectionName); + } + + Map details = properties.getDetails(); + details.put("backupId", backupId.id); + backups.add(details); + } + + results.add("backups", backups); + SolrResponse response = new OverseerSolrResponse(results); + rsp.getValues().addAll(response.getResponse()); + return null; + } + }), CREATESNAPSHOT_OP(CREATESNAPSHOT, (req, rsp, h) -> { req.getParams().required().check(COLLECTION_PROP, CoreAdminParams.COMMIT_NAME); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/RestoreCoreOp.java b/solr/core/src/java/org/apache/solr/handler/admin/RestoreCoreOp.java index 65214c2226a..5fb215125e0 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/RestoreCoreOp.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/RestoreCoreOp.java @@ -37,33 +37,44 @@ class RestoreCoreOp implements CoreAdminHandler.CoreAdminOp { public void execute(CoreAdminHandler.CallInfo it) throws Exception { final SolrParams params = it.req.getParams(); String cname = params.required().get(CoreAdminParams.CORE); - String name = params.required().get(NAME); + String name = params.get(NAME); + String metafile = params.get(CoreAdminParams.SHARD_BACKUP_ID); + String repoName = params.get(CoreAdminParams.BACKUP_REPOSITORY); + + if (metafile == null && name == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Either backupName or metadata file is not specified"); + } ZkController zkController = it.handler.coreContainer.getZkController(); if (zkController == null) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Only valid for SolrCloud"); } - String repoName = params.get(CoreAdminParams.BACKUP_REPOSITORY); - BackupRepository repository = it.handler.coreContainer.newBackupRepository(repoName); + try (BackupRepository repository = it.handler.coreContainer.newBackupRepository(repoName); + SolrCore core = it.handler.coreContainer.getCore(cname)) { - String location = repository.getBackupLocation(params.get(CoreAdminParams.BACKUP_LOCATION)); - if (location == null) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'location' is not specified as a query" - + " parameter or as a default repository property"); - } + String location = repository.getBackupLocation(params.get(CoreAdminParams.BACKUP_LOCATION)); + if (location == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'location' is not specified as a query" + + " parameter or as a default repository property"); + } - URI locationUri = repository.createURI(location); - try (SolrCore core = it.handler.coreContainer.getCore(cname)) { + URI locationUri = repository.createURI(location); CloudDescriptor cd = core.getCoreDescriptor().getCloudDescriptor(); // this core must be the only replica in its shard otherwise // we cannot guarantee consistency between replicas because when we add data (or restore index) to this replica Slice slice = zkController.getClusterState().getCollection(cd.getCollectionName()).getSlice(cd.getShardId()); - if (slice.getReplicas().size() != 1) { + if (slice.getReplicas().size() != 1 && !core.readOnly) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Failed to restore core=" + core.getName() + ", the core must be the only replica in its shard"); + "Failed to restore core=" + core.getName() + ", the core must be the only replica in its shard or it must be read only"); + } + + RestoreCore restoreCore; + if (metafile != null) { + restoreCore = RestoreCore.createWithMetaFile(repository, core, locationUri, metafile); + } else { + restoreCore = RestoreCore.create(repository, core, locationUri, name); } - RestoreCore restoreCore = new RestoreCore(repository, core, locationUri, name); boolean success = restoreCore.doRestore(); if (!success) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Failed to restore core=" + core.getName()); diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCloudIncrementalBackupTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCloudIncrementalBackupTest.java new file mode 100644 index 00000000000..17fbce04f92 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCloudIncrementalBackupTest.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud.api.collections; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.solr.cloud.hdfs.HdfsTestUtil; +import org.apache.solr.common.util.IOUtils; +import org.apache.solr.util.BadHdfsThreadsFilter; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +@ThreadLeakFilters(defaultFilters = true, filters = { + BadHdfsThreadsFilter.class // hdfs currently leaks thread(s) +}) +public class HdfsCloudIncrementalBackupTest extends AbstractIncrementalBackupTest{ + public static final String SOLR_XML = "\n" + + "\n" + + " ${shareSchema:false}\n" + + " ${configSetBaseDir:configsets}\n" + + " ${coreRootDirectory:.}\n" + + "\n" + + " \n" + + " ${urlScheme:}\n" + + " ${socketTimeout:90000}\n" + + " ${connTimeout:15000}\n" + + " \n" + + "\n" + + " \n" + + " 127.0.0.1\n" + + " ${hostPort:8983}\n" + + " ${hostContext:solr}\n" + + " ${solr.zkclienttimeout:30000}\n" + + " ${genericCoreNodeNames:true}\n" + + " 10000\n" + + " ${distribUpdateConnTimeout:45000}\n" + + " ${distribUpdateSoTimeout:340000}\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " hdfs\n" + + " \n" + + " \n" + + " ${solr.hdfs.default.backup.path}\n" + + " ${solr.hdfs.home:}\n" + + " ${solr.hdfs.confdir:}\n" + + " \n" + + " \n" + + " \n" + + "\n"; + + private static MiniDFSCluster dfsCluster; + private static String hdfsUri; + private static FileSystem fs; + + @BeforeClass + public static void setupClass() throws Exception { + dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath()); + hdfsUri = HdfsTestUtil.getURI(dfsCluster); + try { + URI uri = new URI(hdfsUri); + Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster); + fs = FileSystem.get(uri, conf); + + if (fs instanceof DistributedFileSystem) { + // Make sure dfs is not in safe mode + while (((DistributedFileSystem) fs).setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_GET, true)) { + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + Thread.interrupted(); + // continue + } + } + } + + fs.mkdirs(new org.apache.hadoop.fs.Path("/backup")); + } catch (IOException | URISyntaxException e) { + throw new RuntimeException(e); + } + + System.setProperty("solr.hdfs.default.backup.path", "/backup"); + System.setProperty("solr.hdfs.home", hdfsUri + "/solr"); + useFactory("solr.StandardDirectoryFactory"); + + configureCluster(NUM_SHARDS)// nodes + .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) + .withSolrXml(SOLR_XML) + .configure(); + } + + @AfterClass + public static void teardownClass() throws Exception { + IOUtils.closeQuietly(fs); + fs = null; + try { + HdfsTestUtil.teardownClass(dfsCluster); + } finally { + dfsCluster = null; + System.clearProperty("solr.hdfs.home"); + System.clearProperty("solr.hdfs.default.backup.path"); + System.clearProperty("test.build.data"); + System.clearProperty("test.cache.data"); + } + } + + @Override + public String getCollectionNamePrefix() { + return "hdfsbackuprestore"; + } + + @Override + public String getBackupLocation() { + return null; + } +} diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java new file mode 100644 index 00000000000..b247ae68925 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud.api.collections; + +import org.junit.BeforeClass; + +public class LocalFSCloudIncrementalBackupTest extends AbstractIncrementalBackupTest { + public static final String SOLR_XML = "\n" + + "\n" + + " ${shareSchema:false}\n" + + " ${configSetBaseDir:configsets}\n" + + " ${coreRootDirectory:.}\n" + + "\n" + + " \n" + + " ${urlScheme:}\n" + + " ${socketTimeout:90000}\n" + + " ${connTimeout:15000}\n" + + " \n" + + "\n" + + " \n" + + " 127.0.0.1\n" + + " ${hostPort:8983}\n" + + " ${hostContext:solr}\n" + + " ${solr.zkclienttimeout:30000}\n" + + " ${genericCoreNodeNames:true}\n" + + " 10000\n" + + " ${distribUpdateConnTimeout:45000}\n" + + " ${distribUpdateSoTimeout:340000}\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " localfs\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "\n"; + + private static String backupLocation; + + @BeforeClass + public static void setupClass() throws Exception { + configureCluster(NUM_SHARDS)// nodes + .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) + .withSolrXml(SOLR_XML) + .configure(); + + boolean whitespacesInPath = random().nextBoolean(); + if (whitespacesInPath) { + backupLocation = createTempDir("my backup").toAbsolutePath().toString(); + } else { + backupLocation = createTempDir("mybackup").toAbsolutePath().toString(); + } + } + + @Override + public String getCollectionNamePrefix() { + return "backuprestore"; + } + + @Override + public String getBackupLocation() { + return backupLocation; + } + +} diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/PurgeGraphTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/PurgeGraphTest.java new file mode 100644 index 00000000000..c335a180abf --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/PurgeGraphTest.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud.api.collections; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import com.google.common.collect.ObjectArrays; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.cloud.api.collections.DeleteBackupCmd.PurgeGraph; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.apache.solr.handler.IncrementalBackupPaths; +import org.junit.Test; +import org.mockito.stubbing.Answer; + +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class PurgeGraphTest extends SolrTestCaseJ4 { + private static final String[] shardBackupIds = new String[]{"b1_s1", "b1_s2", "b2_s1", "b2_s2", "b3_s1", "b3_s2", "b3_s3"}; + + @Test + public void test() throws URISyntaxException, IOException { + assumeWorkingMockito(); + BackupRepository repository = mock(BackupRepository.class); + IncrementalBackupPaths paths = mock(IncrementalBackupPaths.class); + when(paths.getBackupLocation()).thenReturn(new URI("/temp")); + when(paths.getIndexDir()).thenReturn(new URI("/temp/index")); + when(paths.getShardBackupIdDir()).thenReturn(new URI("/temp/backup_point")); + + PurgeGraph purgeGraph = new PurgeGraph(); + buildCompleteGraph(repository, paths, purgeGraph); + purgeGraph.findDeletableNodes(repository, paths); + + assertEquals(0, purgeGraph.backupIdDeletes.size()); + assertEquals(0, purgeGraph.shardBackupIdDeletes.size()); + assertEquals(0, purgeGraph.indexFileDeletes.size()); + + testDeleteUnreferencedFiles(repository, paths, purgeGraph); + testMissingBackupPointFiles(repository, paths); + testMissingIndexFiles(repository, paths); + } + + private void testMissingIndexFiles(BackupRepository repository, IncrementalBackupPaths paths) throws IOException { + PurgeGraph purgeGraph = new PurgeGraph(); + buildCompleteGraph(repository, paths, purgeGraph); + + Set indexFiles = purgeGraph.indexFileNodeMap.keySet(); + when(repository.listAllOrEmpty(same(paths.getIndexDir()))).thenAnswer((Answer) invocationOnMock -> { + Set newFiles = new HashSet<>(indexFiles); + newFiles.remove("s1_102"); + return newFiles.toArray(new String[0]); + }); + purgeGraph.findDeletableNodes(repository, paths); + + assertEquals(3, purgeGraph.backupIdDeletes.size()); + assertEquals(shardBackupIds.length, purgeGraph.shardBackupIdDeletes.size()); + assertEquals(purgeGraph.indexFileNodeMap.size(), purgeGraph.indexFileDeletes.size() + 1); + + purgeGraph = new PurgeGraph(); + buildCompleteGraph(repository, paths, purgeGraph); + + Set indexFiles2 = purgeGraph.indexFileNodeMap.keySet(); + when(repository.listAllOrEmpty(same(paths.getIndexDir()))).thenAnswer((Answer) invocationOnMock -> { + Set newFiles = new HashSet<>(indexFiles2); + newFiles.remove("s1_101"); + return newFiles.toArray(new String[0]); + }); + purgeGraph.findDeletableNodes(repository, paths); + + assertEquals(2, purgeGraph.backupIdDeletes.size()); + assertEquals(4, purgeGraph.shardBackupIdDeletes.size()); + assertTrue(purgeGraph.indexFileDeletes.contains("s1_100")); + assertFalse(purgeGraph.indexFileDeletes.contains("s1_101")); + } + + private void testMissingBackupPointFiles(BackupRepository repository, IncrementalBackupPaths paths) throws IOException { + PurgeGraph purgeGraph = new PurgeGraph(); + buildCompleteGraph(repository, paths, purgeGraph); + when(repository.listAllOrEmpty(same(paths.getShardBackupIdDir()))).thenAnswer((Answer) + invocationOnMock -> Arrays.copyOfRange(shardBackupIds, 1, shardBackupIds.length) + ); + purgeGraph.findDeletableNodes(repository, paths); + + assertEquals(1, purgeGraph.backupIdDeletes.size()); + assertEquals("b1", purgeGraph.backupIdDeletes.get(0)); + assertEquals(1, purgeGraph.shardBackupIdDeletes.size()); + assertEquals("b1_s2", purgeGraph.shardBackupIdDeletes.get(0)); + assertTrue(purgeGraph.indexFileDeletes.contains("s1_100")); + assertFalse(purgeGraph.indexFileDeletes.contains("s1_101")); + + purgeGraph = new PurgeGraph(); + buildCompleteGraph(repository, paths, purgeGraph); + when(repository.listAllOrEmpty(same(paths.getShardBackupIdDir()))).thenAnswer((Answer) + invocationOnMock -> new String[]{"b1_s1", "b2_s1", "b3_s1", "b3_s2", "b3_s3"} + ); + purgeGraph.findDeletableNodes(repository, paths); + + assertEquals(2, purgeGraph.backupIdDeletes.size()); + assertTrue(purgeGraph.backupIdDeletes.containsAll(Arrays.asList("b1", "b2"))); + assertEquals(2, purgeGraph.shardBackupIdDeletes.size()); + assertTrue(purgeGraph.shardBackupIdDeletes.containsAll(Arrays.asList("b2_s1", "b1_s1"))); + assertTrue(purgeGraph.indexFileDeletes.containsAll(Arrays.asList("s1_100", "s1_101"))); + assertFalse(purgeGraph.indexFileDeletes.contains("s1_102")); + } + + private void testDeleteUnreferencedFiles(BackupRepository repository, IncrementalBackupPaths paths, + PurgeGraph purgeGraph) throws IOException { + buildCompleteGraph(repository, paths, purgeGraph); + String[] unRefBackupPoints = addUnRefFiles(repository, "b4_s", paths.getShardBackupIdDir()); + String[] unRefIndexFiles = addUnRefFiles(repository, "s4_", paths.getIndexDir()); + + purgeGraph.findDeletableNodes(repository, paths); + + assertEquals(0, purgeGraph.backupIdDeletes.size()); + assertEquals(unRefBackupPoints.length, purgeGraph.shardBackupIdDeletes.size()); + assertTrue(purgeGraph.shardBackupIdDeletes.containsAll(Arrays.asList(unRefBackupPoints))); + assertEquals(unRefIndexFiles.length, purgeGraph.indexFileDeletes.size()); + assertTrue(purgeGraph.indexFileDeletes.containsAll(Arrays.asList(unRefIndexFiles))); + } + + private String[] addUnRefFiles(BackupRepository repository, String prefix, URI dir) { + String[] unRefBackupPoints = new String[random().nextInt(10) + 1]; + for (int i = 0; i < unRefBackupPoints.length; i++) { + unRefBackupPoints[i] = prefix + (100 + i); + } + String[] shardBackupIds = repository.listAllOrEmpty(dir); + when(repository.listAllOrEmpty(same(dir))) + .thenAnswer((Answer) invocation + -> ObjectArrays.concat(shardBackupIds, unRefBackupPoints, String.class)); + return unRefBackupPoints; + } + + private void buildCompleteGraph(BackupRepository repository, IncrementalBackupPaths paths, + PurgeGraph purgeGraph) throws IOException { + when(repository.listAllOrEmpty(same(paths.getShardBackupIdDir()))).thenAnswer((Answer) invocationOnMock -> shardBackupIds); + //logical + + for (String shardBackupId : shardBackupIds) { + purgeGraph.addEdge(purgeGraph.getShardBackupIdNode(shardBackupId), + purgeGraph.getBackupIdNode(shardBackupId.substring(0, 2))); + for (int i = 0; i < random().nextInt(30); i++) { + String fileName = shardBackupId.substring(3) + "_" + random().nextInt(15); + purgeGraph.addEdge(purgeGraph.getShardBackupIdNode(shardBackupId), + purgeGraph.getIndexFileNode(fileName)); + } + } + + purgeGraph.addEdge(purgeGraph.getShardBackupIdNode("b1_s1"), + purgeGraph.getIndexFileNode("s1_100")); + + purgeGraph.addEdge(purgeGraph.getShardBackupIdNode("b1_s1"), + purgeGraph.getIndexFileNode("s1_101")); + purgeGraph.addEdge(purgeGraph.getShardBackupIdNode("b2_s1"), + purgeGraph.getIndexFileNode("s1_101")); + + purgeGraph.addEdge(purgeGraph.getShardBackupIdNode("b1_s1"), + purgeGraph.getIndexFileNode("s1_102")); + purgeGraph.addEdge(purgeGraph.getShardBackupIdNode("b2_s1"), + purgeGraph.getIndexFileNode("s1_102")); + purgeGraph.addEdge(purgeGraph.getShardBackupIdNode("b3_s1"), + purgeGraph.getIndexFileNode("s1_102")); + + when(repository.listAllOrEmpty(same(paths.getIndexDir()))).thenAnswer((Answer) invocationOnMock -> + purgeGraph.indexFileNodeMap.keySet().toArray(new String[0])); + } +} diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java index b646e71cbce..c3f9088b666 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java @@ -24,7 +24,6 @@ import java.util.Collection; import java.util.HashMap; import java.util.Map; -import java.util.Properties; import org.apache.lucene.util.QuickPatchThreadsFilter; @@ -45,6 +44,7 @@ import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.backup.BackupManager; +import org.apache.solr.core.backup.BackupProperties; import org.apache.solr.core.backup.repository.HdfsBackupRepository; import org.apache.solr.util.BadHdfsThreadsFilter; import org.junit.AfterClass; @@ -53,10 +53,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF; -import static org.apache.solr.core.backup.BackupManager.BACKUP_NAME_PROP; import static org.apache.solr.core.backup.BackupManager.BACKUP_PROPS_FILE; -import static org.apache.solr.core.backup.BackupManager.COLLECTION_NAME_PROP; import static org.apache.solr.core.backup.BackupManager.CONFIG_STATE_DIR; import static org.apache.solr.core.backup.BackupManager.ZK_STATE_DIR; @@ -194,17 +191,17 @@ protected void testConfigBackupOnly(String configName, String collectionName) th HdfsBackupRepository repo = new HdfsBackupRepository(); repo.init(new NamedList<>(params)); - BackupManager mgr = new BackupManager(repo, solrClient.getZkStateReader()); URI baseLoc = repo.createURI("/backup"); - Properties props = mgr.readBackupProperties(baseLoc, backupName); + BackupManager mgr = BackupManager.forRestore(repo, solrClient.getZkStateReader(), repo.resolve(baseLoc, backupName)); + BackupProperties props = mgr.readBackupProperties(); assertNotNull(props); - assertEquals(collectionName, props.getProperty(COLLECTION_NAME_PROP)); - assertEquals(backupName, props.getProperty(BACKUP_NAME_PROP)); - assertEquals(configName, props.getProperty(COLL_CONF)); + assertEquals(collectionName, props.getCollection()); + assertEquals(backupName, props.getBackupName()); + assertEquals(configName, props.getConfigName()); - DocCollection collectionState = mgr.readCollectionState(baseLoc, backupName, collectionName); + DocCollection collectionState = mgr.readCollectionState(collectionName); assertNotNull(collectionState); assertEquals(collectionName, collectionState.getName()); diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestLocalFSCloudBackupRestore.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestLocalFSCloudBackupRestore.java index b74080aab00..672ba8cd56a 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestLocalFSCloudBackupRestore.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestLocalFSCloudBackupRestore.java @@ -139,7 +139,7 @@ public static class PoinsionedRepository extends LocalFileSystemRepository { public PoinsionedRepository() { super(); } - @Override + @Override public void copyFileFrom(Directory sourceDir, String fileName, URI dest) throws IOException { throw new UnsupportedOperationException(poisioned); } diff --git a/solr/core/src/test/org/apache/solr/core/backup/BackupIdTest.java b/solr/core/src/test/org/apache/solr/core/backup/BackupIdTest.java new file mode 100644 index 00000000000..cbb65cb429b --- /dev/null +++ b/solr/core/src/test/org/apache/solr/core/backup/BackupIdTest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.core.backup; + +import java.util.Optional; + +import org.apache.solr.SolrTestCase; +import org.junit.Test; + +public class BackupIdTest extends SolrTestCase { + + @Test + public void test() { + BackupId backupId = BackupId.findMostRecent(new String[] {"aaa", "baa.properties", "backup.properties", + "backup_1.properties", "backup_2.properties", "backup_neqewq.properties", "backup999.properties"}).get(); + assertEquals("backup_2.properties", backupId.getBackupPropsName()); + backupId = backupId.nextBackupId(); + assertEquals("backup_3.properties", backupId.getBackupPropsName()); + + Optional op = BackupId.findMostRecent(new String[0]); + assertFalse(op.isPresent()); + } +} diff --git a/solr/core/src/test/org/apache/solr/core/backup/repository/HdfsBackupRepository2Test.java b/solr/core/src/test/org/apache/solr/core/backup/repository/HdfsBackupRepository2Test.java new file mode 100644 index 00000000000..6690b4aba1c --- /dev/null +++ b/solr/core/src/test/org/apache/solr/core/backup/repository/HdfsBackupRepository2Test.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.core.backup.repository; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.solr.cloud.api.collections.AbstractBackupRepositoryTest; +import org.apache.solr.cloud.hdfs.HdfsTestUtil; +import org.apache.solr.common.util.IOUtils; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.HdfsDirectoryFactory; +import org.apache.solr.util.BadHdfsThreadsFilter; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +@ThreadLeakFilters(defaultFilters = true, filters = { + BadHdfsThreadsFilter.class // hdfs currently leaks thread(s) +}) +public class HdfsBackupRepository2Test extends AbstractBackupRepositoryTest { + private static MiniDFSCluster dfsCluster; + private static String hdfsUri; + private static FileSystem fs; + + @BeforeClass + public static void setupClass() throws Exception { + dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath()); + hdfsUri = HdfsTestUtil.getURI(dfsCluster); + try { + URI uri = new URI(hdfsUri); + Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster); + fs = FileSystem.get(uri, conf); + + if (fs instanceof DistributedFileSystem) { + // Make sure dfs is not in safe mode + while (((DistributedFileSystem) fs).setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_GET, true)) { + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + Thread.interrupted(); + // continue + } + } + } + + fs.mkdirs(new org.apache.hadoop.fs.Path("/backup")); + } catch (IOException | URISyntaxException e) { + throw new RuntimeException(e); + } + + System.setProperty("solr.hdfs.default.backup.path", "/backup"); + System.setProperty("solr.hdfs.home", hdfsUri + "/solr"); + useFactory("solr.StandardDirectoryFactory"); + } + + @AfterClass + public static void teardownClass() throws Exception { + IOUtils.closeQuietly(fs); + fs = null; + try { + HdfsTestUtil.teardownClass(dfsCluster); + } finally { + dfsCluster = null; + System.clearProperty("solr.hdfs.home"); + System.clearProperty("solr.hdfs.default.backup.path"); + System.clearProperty("test.build.data"); + System.clearProperty("test.cache.data"); + } + } + + @Override + @SuppressWarnings({"rawtypes", "unchecked"}) + protected BackupRepository getRepository() { + HdfsBackupRepository repository = new HdfsBackupRepository(); + NamedList config = new NamedList(); + config.add(HdfsDirectoryFactory.HDFS_HOME, hdfsUri + "/solr"); + repository.init(config); + return repository; + } + + @Override + protected URI getBaseUri() throws URISyntaxException { + return new URI(hdfsUri+"/solr/tmp"); + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/TestStressIncrementalBackup.java b/solr/core/src/test/org/apache/solr/handler/TestStressIncrementalBackup.java new file mode 100644 index 00000000000..871d3498be6 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/TestStressIncrementalBackup.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler; + +import java.io.File; +import java.lang.invoke.MethodHandles; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.lucene.util.LuceneTestCase; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.response.CollectionAdminResponse; +import org.apache.solr.client.solrj.response.RequestStatusState; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.params.UpdateParams; +import org.apache.solr.util.LogLevel; +import org.junit.After; +import org.junit.Before; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.handler.TestStressThreadBackup.makeDoc; + +//@LuceneTestCase.Nightly +@LuceneTestCase.SuppressCodecs({"SimpleText"}) +@LogLevel("org.apache.solr.handler.SnapShooter=DEBUG;org.apache.solr.core.IndexDeletionPolicyWrapper=DEBUG") +public class TestStressIncrementalBackup extends SolrCloudTestCase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private File backupDir; + private SolrClient adminClient; + private SolrClient coreClient; + @Before + public void beforeTest() throws Exception { + backupDir = createTempDir(getTestClass().getSimpleName() + "_backups").toFile(); + + // NOTE: we don't actually care about using SolrCloud, but we want to use SolrClient and I can't + // bring myself to deal with the nonsense that is SolrJettyTestBase. + + // We do however explicitly want a fresh "cluster" every time a test is run + configureCluster(1) + .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) + .configure(); + + assertEquals(0, (CollectionAdminRequest.createCollection(DEFAULT_TEST_COLLECTION_NAME, "conf1", 1, 1) + .process(cluster.getSolrClient()).getStatus())); + adminClient = getHttpSolrClient(cluster.getJettySolrRunners().get(0).getBaseUrl().toString()); + initCoreNameAndSolrCoreClient(); + } + + private void initCoreNameAndSolrCoreClient() { + // Sigh. + Replica r = cluster.getSolrClient().getZkStateReader().getClusterState() + .getCollection(DEFAULT_TEST_COLLECTION_NAME).getActiveSlices().iterator().next() + .getReplicas().iterator().next(); + coreName = r.getCoreName(); + coreClient = getHttpSolrClient(r.getCoreUrl()); + } + + @After + public void afterTest() throws Exception { + // we use a clean cluster instance for every test, so we need to clean it up + shutdownCluster(); + + if (null != adminClient) { + adminClient.close(); + } + if (null != coreClient) { + coreClient.close(); + } + } + + public void testCoreAdminHandler() throws Exception { + final int numBackupIters = 20; // don't use 'atLeast', we don't want to blow up on nightly + + final AtomicReference heavyCommitFailure = new AtomicReference<>(); + final AtomicBoolean keepGoing = new AtomicBoolean(true); + + // this thread will do nothing but add/commit new 'dummy' docs over and over again as fast as possible + // to create a lot of index churn w/ segment merging + final Thread heavyCommitting = new Thread() { + public void run() { + try { + int docIdCounter = 0; + while (keepGoing.get()) { + docIdCounter++; + + final UpdateRequest req = new UpdateRequest().add(makeDoc("dummy_" + docIdCounter, "dummy")); + // always commit to force lots of new segments + req.setParam(UpdateParams.COMMIT,"true"); + req.setParam(UpdateParams.OPEN_SEARCHER,"false"); // we don't care about searching + + // frequently forceMerge to ensure segments are frequently deleted + if (0 == (docIdCounter % 13)) { // arbitrary + req.setParam(UpdateParams.OPTIMIZE, "true"); + req.setParam(UpdateParams.MAX_OPTIMIZE_SEGMENTS, "5"); // arbitrary + } + + log.info("Heavy Committing #{}: {}", docIdCounter, req); + final UpdateResponse rsp = req.process(coreClient); + assertEquals("Dummy Doc#" + docIdCounter + " add status: " + rsp.toString(), 0, rsp.getStatus()); + + } + } catch (Throwable t) { + heavyCommitFailure.set(t); + } + } + }; + + heavyCommitting.start(); + try { + // now have the "main" test thread try to take a serious of backups/snapshots + // while adding other "real" docs + + // NOTE #1: start at i=1 for 'id' & doc counting purposes... + // NOTE #2: abort quickly if the oher thread reports a heavyCommitFailure... + for (int i = 1; (i <= numBackupIters && null == heavyCommitFailure.get()); i++) { + + // in each iteration '#i', the commit we create should have exactly 'i' documents in + // it with the term 'type_s:real' (regardless of what the other thread does with dummy docs) + + // add & commit a doc #i + final UpdateRequest req = new UpdateRequest().add(makeDoc("doc_" + i, "real")); + req.setParam(UpdateParams.COMMIT,"true"); // make immediately available for backup + req.setParam(UpdateParams.OPEN_SEARCHER,"false"); // we don't care about searching + + final UpdateResponse rsp = req.process(coreClient); + assertEquals("Real Doc#" + i + " add status: " + rsp.toString(), 0, rsp.getStatus()); + + makeBackup(); + } + + } finally { + keepGoing.set(false); + heavyCommitting.join(); + } + assertNull(heavyCommitFailure.get()); + } + + public void makeBackup() throws Exception { + CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(DEFAULT_TEST_COLLECTION_NAME, "stressBackup") + .setLocation(backupDir.getAbsolutePath()) + .setIncremental(true) + .setMaxNumberBackupPoints(5); + if (random().nextBoolean()) { + try { + RequestStatusState state = backup.processAndWait(cluster.getSolrClient(), 1000); + assertEquals(RequestStatusState.COMPLETED, state); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } else { + CollectionAdminResponse rsp = backup.process(cluster.getSolrClient()); + assertEquals(0, rsp.getStatus()); + } + } + +} diff --git a/solr/core/src/test/org/apache/solr/handler/TestStressThreadBackup.java b/solr/core/src/test/org/apache/solr/handler/TestStressThreadBackup.java index 3622948c50f..3113ee43eb2 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestStressThreadBackup.java +++ b/solr/core/src/test/org/apache/solr/handler/TestStressThreadBackup.java @@ -54,10 +54,7 @@ import org.apache.solr.common.SolrInputDocument; import org.apache.solr.util.TimeOut; import org.apache.solr.util.LogLevel; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; +import org.junit.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -114,11 +111,13 @@ public void afterTest() throws Exception { } } + @Test public void testCoreAdminHandler() throws Exception { // Use default BackupAPIImpl which hits CoreAdmin API for everything testSnapshotsAndBackupsDuringConcurrentCommitsAndOptimizes(new BackupAPIImpl()); } - + + @Test public void testReplicationHandler() throws Exception { // Create a custom BackupAPIImpl which uses ReplicatoinHandler for the backups // but still defaults to CoreAdmin for making named snapshots (since that's what's documented) @@ -329,7 +328,7 @@ private void validateBackup(final File backup) throws IOException { * @param id the uniqueKey * @param type the type of the doc for use in the 'type_s' field (for term counting later) */ - private static SolrInputDocument makeDoc(String id, String type) { + static SolrInputDocument makeDoc(String id, String type) { final SolrInputDocument doc = new SolrInputDocument("id", id, "type_s", type); for (int f = 0; f < 100; f++) { doc.addField(f + "_s", TestUtil.randomUnicodeString(random(), 20)); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java index 44454e0e146..6e1d4dd65c7 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java @@ -975,6 +975,8 @@ public static class Backup extends AsyncCollectionSpecificAdminRequest { protected String location; protected Optional commitName = Optional.empty(); protected Optional indexBackupStrategy = Optional.empty(); + protected boolean incremental = false; + protected Optional maxNumBackupPoints = Optional.empty(); public Backup(String collection, String name) { super(CollectionAction.BACKUP, collection); @@ -1018,6 +1020,16 @@ public Backup setIndexBackupStrategy(String indexBackupStrategy) { return this; } + public Backup setIncremental(boolean incremental) { + this.incremental = incremental; + return this; + } + + public Backup setMaxNumberBackupPoints(int maxNumBackupPoints) { + this.maxNumBackupPoints = Optional.of(maxNumBackupPoints); + return this; + } + @Override public SolrParams getParams() { ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); @@ -1033,6 +1045,10 @@ public SolrParams getParams() { if (indexBackupStrategy.isPresent()) { params.set(CollectionAdminParams.INDEX_BACKUP_STRATEGY, indexBackupStrategy.get()); } + if (maxNumBackupPoints.isPresent()) { + params.set(CoreAdminParams.MAX_NUM_BACKUP, maxNumBackupPoints.get()); + } + params.set(CoreAdminParams.BACKUP_INCREMENTAL, incremental); return params; } @@ -1057,6 +1073,7 @@ public static class Restore extends AsyncCollectionSpecificAdminRequest { protected Optional createNodeSet = Optional.empty(); protected Optional createNodeSetShuffle = Optional.empty(); protected Properties properties; + protected Integer backupId; public Restore(String collection, String backupName) { super(CollectionAction.RESTORE, collection); @@ -1118,6 +1135,11 @@ public Properties getProperties() { } public Restore setProperties(Properties properties) { this.properties = properties; return this;} + public Restore setBackupId(int backupId) { + this.backupId = backupId; + return this; + } + // TODO support rule, snitch @Override @@ -1155,6 +1177,9 @@ public SolrParams getParams() { if (createNodeSetShuffle.isPresent()) { params.set(CREATE_NODE_SET_SHUFFLE_PARAM, createNodeSetShuffle.get()); } + if (backupId != null) { + params.set(CoreAdminParams.BACKUP_ID, backupId); + } return params; } @@ -2659,6 +2684,87 @@ protected CollectionAdminResponse createResponse(SolrClient client) { } } + // DELETEBACKUP request + public static class DeleteBackup extends CollectionAdminRequest { + private String backupName; + private String backupRepo; + private String backupLocation; + private Boolean purge; + private Integer backupId; + private Integer keepLastNumberOfBackups; + + public DeleteBackup (String backupRepo, String backupLocation, String backupName) { + super(CollectionAction.DELETEBACKUP); + this.backupName = backupName; + this.backupRepo = backupRepo; + this.backupLocation = backupLocation; + } + + public DeleteBackup deleteBackupId(int backupId) { + this.backupId = backupId; + return this; + } + + public DeleteBackup keepLastNumberOfBackups(int num) { + this.keepLastNumberOfBackups = num; + return this; + } + + public DeleteBackup purgeBackups(boolean purge) { + this.purge = purge; + return this; + } + + @Override + public SolrParams getParams() { + ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); + params.set(CoreAdminParams.NAME, backupName); + params.set(CoreAdminParams.BACKUP_LOCATION, backupLocation); + params.set(CoreAdminParams.BACKUP_REPOSITORY, backupRepo); + if (backupId != null) + params.set(CoreAdminParams.BACKUP_ID, backupId); + if (keepLastNumberOfBackups != null) + params.set(CoreAdminParams.MAX_NUM_BACKUP, keepLastNumberOfBackups); + if (purge != null) + params.set(CoreAdminParams.PURGE_BACKUP, purge); + return params; + } + + @Override + protected CollectionAdminResponse createResponse(SolrClient client) { + return new CollectionAdminResponse(); + } + } + + // LISTBACKUP request + public static class ListBackup extends CollectionAdminRequest { + private String backupName; + private String backupRepo; + private String backupLocation; + + public ListBackup(String backupRepo, String backupLocation, String backupName) { + super(CollectionAction.LISTBACKUP); + this.backupName = backupName; + this.backupRepo = backupRepo; + this.backupLocation = backupLocation; + } + + @Override + public SolrParams getParams() { + ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); + params.set(CoreAdminParams.NAME, backupName); + params.set(CoreAdminParams.BACKUP_LOCATION, backupLocation); + params.set(CoreAdminParams.BACKUP_REPOSITORY, backupRepo); + + return params; + } + + @Override + protected CollectionAdminResponse createResponse(SolrClient client) { + return new CollectionAdminResponse(); + } + } + /** * Returns a SolrRequest to add a property to a specific replica */ diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java index f55cb70e2f5..1ffa05e7980 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java @@ -113,6 +113,8 @@ enum CollectionAction { MODIFYCOLLECTION(true, LockLevel.COLLECTION), BACKUP(true, LockLevel.COLLECTION), RESTORE(true, LockLevel.COLLECTION), + LISTBACKUP(false, LockLevel.NONE), + DELETEBACKUP(true, LockLevel.COLLECTION), CREATESNAPSHOT(true, LockLevel.COLLECTION), DELETESNAPSHOT(true, LockLevel.COLLECTION), LISTSNAPSHOTS(false, LockLevel.NONE), diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java index 2548e621315..05d6d08d153 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java @@ -122,6 +122,36 @@ public abstract class CoreAdminParams */ public static final String BACKUP_LOCATION = "location"; + /** + * A parameter to specify the name of previous shardBackupId. + */ + public static final String PREV_SHARD_BACKUP_ID = "prevShardBackupId"; + + /** + * A parameter to specify the name of shardBackupId which will be created + */ + public static final String SHARD_BACKUP_ID = "shardBackupId"; + + /** + * A parameter to specify last number of backups (delete the rest) + */ + public static final String MAX_NUM_BACKUP = "maxNumBackup"; + + /** + * Unique id of the backup + */ + public static final String BACKUP_ID = "backupId"; + + /** + * Purging/deleting all indexFiles, shardBackupIds, backupIds that are unreachable, uncompleted or corrupted. + */ + public static final String PURGE_BACKUP = "purge"; + + /** + * A parameter to specify whether incremental backup is used + */ + public static final String BACKUP_INCREMENTAL = "incremental"; + /** * A parameter to specify the name of the commit to be stored during the backup operation. */ diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractBackupRepositoryTest.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractBackupRepositoryTest.java new file mode 100644 index 00000000000..eb46d778adc --- /dev/null +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractBackupRepositoryTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud.api.collections; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.NoSuchFileException; +import java.util.Arrays; +import java.util.HashSet; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.junit.Test; + +public abstract class AbstractBackupRepositoryTest extends SolrTestCaseJ4 { + + protected abstract BackupRepository getRepository(); + + protected abstract URI getBaseUri() throws URISyntaxException; + + @Test + public void test() throws IOException, URISyntaxException { + try (BackupRepository repo = getRepository()) { + URI baseUri = repo.resolve(getBaseUri(), "tmp"); + if (repo.exists(baseUri)) { + repo.deleteDirectory(baseUri); + } + assertFalse(repo.exists(baseUri)); + repo.createDirectory(baseUri); + assertTrue(repo.exists(baseUri)); + assertEquals(0, repo.listAll(baseUri).length); + + // test nested structure + URI tmpFolder = repo.resolve(baseUri, "tmpDir"); + repo.createDirectory(tmpFolder); + assertEquals(repo.getPathType(tmpFolder), BackupRepository.PathType.DIRECTORY); + addFile(repo, repo.resolve(tmpFolder, "file1")); + addFile(repo, repo.resolve(tmpFolder, "file2")); + assertEquals(repo.getPathType(repo.resolve(tmpFolder, "file1")), BackupRepository.PathType.FILE); + String[] files = repo.listAll(tmpFolder); + assertEquals(new HashSet<>(Arrays.asList("file1", "file2")), new HashSet<>(Arrays.asList(files))); + + URI tmpFolder2 = repo.resolve(tmpFolder, "tmpDir2"); + repo.createDirectory(tmpFolder2); + addFile(repo, repo.resolve(tmpFolder2, "file3")); + addFile(repo, repo.resolve(tmpFolder2, "file4")); + addFile(repo, repo.resolve(tmpFolder2, "file5")); + //2 files + 1 folder + assertEquals(3, repo.listAll(tmpFolder).length); + // create same directory must be a no-op + repo.createDirectory(tmpFolder2); + assertEquals(3, repo.listAll(tmpFolder2).length); + assertTrue(repo.exists(tmpFolder2)); + assertTrue(repo.exists(repo.resolve(tmpFolder2, "file3"))); + try { + repo.delete(tmpFolder2, Arrays.asList("file7", "file6"), false); + fail("Delete non existence file leads to success"); + } catch (NoSuchFileException e) { + // expected + } + repo.delete(tmpFolder2, Arrays.asList("file7", "file6"), true); + repo.delete(tmpFolder2, Arrays.asList("file3", "file4"), true); + assertEquals(1, repo.listAll(tmpFolder2).length); + assertFalse(repo.exists(repo.resolve(tmpFolder2, "file3"))); + repo.deleteDirectory(tmpFolder); + assertFalse(repo.exists(tmpFolder)); + assertFalse(repo.exists(repo.resolve(tmpFolder2, "file5"))); + assertFalse(repo.exists(repo.resolve(tmpFolder, "file1"))); + } + } + + private void addFile(BackupRepository repo, URI file) throws IOException { + try (OutputStream os = repo.createOutput(file)) { + os.write(100); + os.write(101); + os.write(102); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java similarity index 99% rename from solr/core/src/test/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java rename to solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java index ccfc78dc9ee..d62b98f6447 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java @@ -400,7 +400,7 @@ private void testBackupAndRestore(String collectionName, int backupReplFactor) t // TODO Find the applicable core.properties on the file system but how? } - private Map getShardToDocCountMap(CloudSolrClient client, DocCollection docCollection) throws SolrServerException, IOException { + public static Map getShardToDocCountMap(CloudSolrClient client, DocCollection docCollection) throws SolrServerException, IOException { Map shardToDocCount = new TreeMap<>(); for (Slice slice : docCollection.getActiveSlices()) { String shardName = slice.getName(); diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java new file mode 100644 index 00000000000..58094dfe360 --- /dev/null +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java @@ -0,0 +1,535 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud.api.collections; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.lang.invoke.MethodHandles; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.commons.io.IOUtils; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.TestUtil; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.response.CollectionAdminResponse; +import org.apache.solr.client.solrj.response.RequestStatusState; +import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.DirectoryFactory; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.TrackingBackupRepository; +import org.apache.solr.core.backup.BackupId; +import org.apache.solr.core.backup.BackupProperties; +import org.apache.solr.core.backup.Checksum; +import org.apache.solr.core.backup.ShardBackupId; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.apache.solr.handler.IncrementalBackupPaths; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.core.TrackingBackupRepository.copiedFiles; + +public abstract class AbstractIncrementalBackupTest extends SolrCloudTestCase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static long docsSeed; // see indexDocs() + protected static final int NUM_SHARDS = 2;//granted we sometimes shard split to get more + protected static final String BACKUPNAME_PREFIX = "mytestbackup"; + protected static final String BACKUP_REPO_NAME = "trackingBackupRepository"; + + protected String testSuffix = "test1"; + protected int replFactor; + protected int numTlogReplicas; + protected int numPullReplicas; + + @BeforeClass + public static void createCluster() throws Exception { + docsSeed = random().nextLong(); + System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory"); + } + + /** + * @return The name of the collection to use. + */ + public abstract String getCollectionNamePrefix(); + + public String getCollectionName(){ + return getCollectionNamePrefix() + "_" + testSuffix; + } + + public void setTestSuffix(String testSuffix) { + this.testSuffix = testSuffix; + } + + private void randomizeReplicaTypes() { + replFactor = TestUtil.nextInt(random(), 1, 2); +// numTlogReplicas = TestUtil.nextInt(random(), 0, 1); +// numPullReplicas = TestUtil.nextInt(random(), 0, 1); + } + + /** + * @return The absolute path for the backup location. + * Could return null. + */ + public abstract String getBackupLocation(); + + @Test + public void testSimple() throws Exception { + TrackingBackupRepository.clear(); + + setTestSuffix("testbackupincsimple"); + CloudSolrClient solrClient = cluster.getSolrClient(); + + CollectionAdminRequest + .createCollection(getCollectionName(), "conf1", NUM_SHARDS, 1) + .process(solrClient); + int numDocs = indexDocs(getCollectionName(), true); + String backupName = BACKUPNAME_PREFIX + testSuffix; + try (BackupRepository repository = cluster.getJettySolrRunner(0).getCoreContainer() + .newBackupRepository(BACKUP_REPO_NAME)) { + String backupLocation = repository.getBackupLocation(getBackupLocation()); + long t = System.nanoTime(); + CollectionAdminRequest.backupCollection(getCollectionName(), backupName) + .setLocation(backupLocation) + .setIncremental(true) + .setRepositoryName(BACKUP_REPO_NAME) + .processAndWait(cluster.getSolrClient(), 100); + long timeTaken = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - t); + log.info("Created backup with {} docs, took {}ms", numDocs, timeTaken); + indexDocs(getCollectionName(), true); + + t = System.nanoTime(); + CollectionAdminRequest.backupCollection(getCollectionName(), backupName) + .setLocation(backupLocation) + .setIncremental(true) + .setRepositoryName(BACKUP_REPO_NAME) + .processAndWait(cluster.getSolrClient(), 100); + timeTaken = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - t); + long numFound = cluster.getSolrClient().query(getCollectionName(), + new SolrQuery("*:*")).getResults().getNumFound(); + log.info("Created backup with {} docs, took {}ms", numFound, timeTaken); + + t = System.nanoTime(); + CollectionAdminRequest.restoreCollection(getCollectionName(), backupName) + .setBackupId(0) + .setLocation(backupLocation).setRepositoryName(BACKUP_REPO_NAME).processAndWait(solrClient, 100); + timeTaken = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - t); + log.info("Restored from backup, took {}ms", timeTaken); + numFound = cluster.getSolrClient().query(getCollectionName(), + new SolrQuery("*:*")).getResults().getNumFound(); + assertEquals(numDocs, numFound); + } + } + + @Test + @Slow + @SuppressWarnings("rawtypes") + public void testBackupIncremental() throws Exception { + TrackingBackupRepository.clear(); + + setTestSuffix("testbackupinc"); + randomizeReplicaTypes(); + CloudSolrClient solrClient = cluster.getSolrClient(); + + CollectionAdminRequest + .createCollection(getCollectionName(), "conf1", NUM_SHARDS, replFactor, numTlogReplicas, numPullReplicas) + .process(solrClient); + + indexDocs(getCollectionName(), false); + + String backupName = BACKUPNAME_PREFIX + testSuffix; + try (BackupRepository repository = cluster.getJettySolrRunner(0).getCoreContainer() + .newBackupRepository(BACKUP_REPO_NAME)) { + String backupLocation = repository.getBackupLocation(getBackupLocation()); + URI uri = repository.resolve(repository.createURI(backupLocation), backupName); + IncrementalBackupPaths backupPaths = new IncrementalBackupPaths(repository, uri); + IncrementalBackupVerifier verifier = new IncrementalBackupVerifier(repository, backupLocation, backupName, 3); + + backupRestoreThenCheck(solrClient, verifier); + indexDocs(getCollectionName(), false); + backupRestoreThenCheck(solrClient, verifier); + + // adding more commits to trigger merging segments + for (int i = 0; i < 15; i++) { + indexDocs(getCollectionName(), 5,false); + } + backupRestoreThenCheck(solrClient, verifier); + + indexDocs(getCollectionName(), false); + backupRestoreThenCheck(solrClient, verifier); + + // test list backups + CollectionAdminResponse resp = + new CollectionAdminRequest.ListBackup(BACKUP_REPO_NAME, backupLocation, backupName).process(cluster.getSolrClient()); + ArrayList backups = (ArrayList) resp.getResponse().get("backups"); + assertEquals(3, backups.size()); + + // test delete backups + resp = new CollectionAdminRequest.DeleteBackup(BACKUP_REPO_NAME, backupLocation, backupName) + .keepLastNumberOfBackups(4) + .process(cluster.getSolrClient()); + assertEquals(null, resp.getResponse().get("deleted")); + + resp = new CollectionAdminRequest.DeleteBackup(BACKUP_REPO_NAME, backupLocation, backupName) + .keepLastNumberOfBackups(3) + .process(cluster.getSolrClient()); + assertEquals(null, resp.getResponse().get("deleted")); + + resp = new CollectionAdminRequest.DeleteBackup(BACKUP_REPO_NAME, backupLocation, backupName) + .keepLastNumberOfBackups(2) + .process(cluster.getSolrClient()); + assertEquals(1, resp.getResponse()._get("deleted[0]/backupId", null)); + + resp = new CollectionAdminRequest.DeleteBackup(BACKUP_REPO_NAME, backupLocation, backupName) + .deleteBackupId(3) + .process(cluster.getSolrClient()); + assertEquals(3, resp.getResponse()._get("deleted[0]/backupId", null)); + + simpleRestoreAndCheckDocCount(solrClient, backupLocation, backupName); + + // test purge backups + // purging first since there may corrupted files were uploaded + resp = new CollectionAdminRequest.DeleteBackup(BACKUP_REPO_NAME, backupLocation, backupName) + .purgeBackups(true) + .process(cluster.getSolrClient()); + + addDummyFileToIndex(repository, backupPaths.getIndexDir(), "dummy-files-1"); + addDummyFileToIndex(repository, backupPaths.getIndexDir(), "dummy-files-2"); + resp = new CollectionAdminRequest.DeleteBackup(BACKUP_REPO_NAME, backupLocation, backupName) + .purgeBackups(true) + .process(cluster.getSolrClient()); + assertEquals(2, ((NamedList)resp.getResponse().get("deleted")).get("numIndexFiles")); + + new UpdateRequest() + .deleteByQuery("*:*") + .commit(cluster.getSolrClient(), getCollectionName()); + indexDocs(getCollectionName(), false); + // corrupt index files + corruptIndexFiles(); + try { + log.info("Create backup after corrupt index files"); + CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(getCollectionName(), backupName) + .setLocation(backupLocation) + .setIncremental(true) + .setMaxNumberBackupPoints(3) + .setRepositoryName(BACKUP_REPO_NAME); + if (random().nextBoolean()) { + RequestStatusState state = backup.processAndWait(cluster.getSolrClient(), 1000); + if (state != RequestStatusState.FAILED) { + fail("This backup should be failed"); + } + } else { + CollectionAdminResponse rsp = backup.process(cluster.getSolrClient()); + fail("This backup should be failed"); + } + } catch (Exception e) { + // expected + e.printStackTrace(); + } + } + } + + protected void corruptIndexFiles() throws IOException { + Collection slices = getCollectionState(getCollectionName()).getSlices(); + Slice slice = slices.iterator().next(); + JettySolrRunner leaderNode = cluster.getReplicaJetty(slice.getLeader()); + + SolrCore solrCore = leaderNode.getCoreContainer().getCore(slice.getLeader().getCoreName()); + Set fileNames = new HashSet<>(solrCore.getDeletionPolicy().getLatestCommit().getFileNames()); + File indexFolder = new File(solrCore.getIndexDir()); + File fileGetCorrupted = Stream.of(Objects.requireNonNull(indexFolder.listFiles())) + .filter(x -> fileNames.contains(x.getName())) + .findAny().get(); + try (FileInputStream fis = new FileInputStream(fileGetCorrupted)){ + byte[] contents = IOUtils.readFully(fis, (int) fileGetCorrupted.length()); + contents[contents.length - CodecUtil.footerLength() - 1] += 1; + contents[contents.length - CodecUtil.footerLength() - 2] += 1; + contents[contents.length - CodecUtil.footerLength() - 3] += 1; + contents[contents.length - CodecUtil.footerLength() - 4] += 1; + try (FileOutputStream fos = new FileOutputStream(fileGetCorrupted)) { + IOUtils.write(contents, fos); + } + } finally { + solrCore.close(); + } + } + + + private void addDummyFileToIndex(BackupRepository repository, URI indexDir, String fileName) throws IOException { + try (OutputStream os = repository.createOutput(repository.resolve(indexDir, fileName))){ + os.write(100); + os.write(101); + os.write(102); + } + } + + private void backupRestoreThenCheck(CloudSolrClient solrClient, + IncrementalBackupVerifier verifier) throws Exception { + verifier.incrementalBackupThenVerify(); + + if( random().nextBoolean() ) + simpleRestoreAndCheckDocCount(solrClient, verifier.backupLocation, verifier.backupName); + } + + private void simpleRestoreAndCheckDocCount(CloudSolrClient solrClient, String backupLocation, String backupName) throws Exception{ + Map origShardToDocCount = AbstractCloudBackupRestoreTestCase.getShardToDocCountMap(solrClient, getCollectionState(getCollectionName())); + + boolean onDiffCollection = random().nextBoolean(); + String restoreCollectionName = getCollectionName(); + if (onDiffCollection) { + restoreCollectionName = getCollectionName() + "_restored"; + } + + CollectionAdminRequest.restoreCollection(restoreCollectionName, backupName) + .setLocation(backupLocation).setRepositoryName(BACKUP_REPO_NAME).process(solrClient); + + AbstractDistribZkTestBase.waitForRecoveriesToFinish( + restoreCollectionName, cluster.getSolrClient().getZkStateReader(), log.isDebugEnabled(), true, 30); + + // check num docs are the same + assertEquals(origShardToDocCount, AbstractCloudBackupRestoreTestCase.getShardToDocCountMap(solrClient, getCollectionState(restoreCollectionName))); + + // this methods may get invoked multiple times, collection must be cleanup + if (onDiffCollection) { + CollectionAdminRequest.deleteCollection(restoreCollectionName).process(solrClient); + } + } + + + private void indexDocs(String collectionName, int numDocs, boolean useUUID) throws Exception { + Random random = new Random(docsSeed); + + List docs = new ArrayList<>(numDocs); + for (int i=0; i> lastShardCommitToBackupFiles = new HashMap<>(); + // the first generation after calling backup is zero + private int numBackup = -1; + private int maxNumberOfBackupToKeep = 4; + + IncrementalBackupVerifier(BackupRepository repository, String backupLocation, + String backupName, int maxNumberOfBackupToKeep) { + this.repository = repository; + this.backupLocation = backupLocation; + this.backupURI = repository.resolve(repository.createURI(backupLocation), backupName); + this.incBackupFiles = new IncrementalBackupPaths(repository, this.backupURI); + this.backupName = backupName; + this.maxNumberOfBackupToKeep = maxNumberOfBackupToKeep; + } + + @SuppressWarnings("rawtypes") + private void backupThenWait() throws SolrServerException, IOException { + CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(getCollectionName(), backupName) + .setLocation(backupLocation) + .setIncremental(true) + .setMaxNumberBackupPoints(maxNumberOfBackupToKeep) + .setRepositoryName(BACKUP_REPO_NAME); + if (random().nextBoolean()) { + try { + RequestStatusState state = backup.processAndWait(cluster.getSolrClient(), 1000); + assertEquals(RequestStatusState.COMPLETED, state); + } catch (InterruptedException e) { + e.printStackTrace(); + } + numBackup++; + } else { + CollectionAdminResponse rsp = backup.process(cluster.getSolrClient()); + assertEquals(0, rsp.getStatus()); + NamedList resp = (NamedList) rsp.getResponse().get("response"); + numBackup++; + assertEquals(numBackup, resp.get("backupId"));; + } + } + + void incrementalBackupThenVerify() throws IOException, SolrServerException { + int numCopiedFiles = copiedFiles().size(); + backupThenWait(); + List newFilesCopiedOver = copiedFiles().subList(numCopiedFiles, copiedFiles().size()); + verify(newFilesCopiedOver); + } + + ShardBackupId getLastShardBackupId(String shardName) throws IOException { + String metaFile = BackupProperties + .readFromLatest(repository, backupURI) + .flatMap(bp -> bp.getShardBackupIdFor(shardName)) + .get(); + return ShardBackupId.from(repository, new IncrementalBackupPaths(repository, backupURI).getShardBackupIdDir(), metaFile); + } + + private void assertIndexInputEquals(IndexInput in1, IndexInput in2) throws IOException { + assertEquals(in1.length(), in2.length()); + for (int i = 0; i < in1.length(); i++) { + assertEquals(in1.readByte(), in2.readByte()); + } + } + + private void assertFolderAreSame(URI uri1, URI uri2) throws IOException { + String[] files1 = repository.listAll(uri1); + String[] files2 = repository.listAll(uri2); + Arrays.sort(files1); + Arrays.sort(files2); + + try { + assertArrayEquals(files1, files2); + } catch (AssertionError e) { + e.printStackTrace(); + } + + for (int i = 0; i < files1.length; i++) { + URI file1Uri = repository.resolve(uri1, files1[i]); + URI file2Uri = repository.resolve(uri2, files2[i]); + assertEquals(repository.getPathType(file1Uri), repository.getPathType(file2Uri)); + if (repository.getPathType(file1Uri) == BackupRepository.PathType.DIRECTORY) { + assertFolderAreSame(file1Uri, file2Uri); + } else { + try (IndexInput in1 = repository.openInput(uri1, files1[i], IOContext.READONCE); + IndexInput in2 = repository.openInput(uri1, files1[i], IOContext.READONCE)) { + assertIndexInputEquals(in1, in2); + } + } + } + } + + public void verify(List newFilesCopiedOver) throws IOException { + //Verify zk files are reuploaded to a appropriate each time a backup is called + //TODO make a little change to zk files and make sure that backed up files match with zk data + BackupId prevBackupId = new BackupId(Math.max(0, numBackup - 1)); + + URI backupPropertiesFile = repository.resolve(backupURI, "backup_"+numBackup+".properties"); + URI zkBackupFolder = repository.resolve(backupURI, "zk_backup_"+numBackup); + assertTrue(repository.exists(backupPropertiesFile)); + assertTrue(repository.exists(zkBackupFolder)); + assertFolderAreSame(repository.resolve(backupURI, prevBackupId.getZkStateDir()), zkBackupFolder); + + // verify indexes file + for(Slice slice : getCollectionState(getCollectionName()).getSlices()) { + Replica leader = slice.getLeader(); + final ShardBackupId shardBackupId = getLastShardBackupId(slice.getName()); + + try (SolrCore solrCore = cluster.getReplicaJetty(leader).getCoreContainer().getCore(leader.getCoreName())) { + Directory dir = solrCore.getDirectoryFactory().get(solrCore.getIndexDir(), DirectoryFactory.DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType); + try { + URI indexDir = incBackupFiles.getIndexDir(); + IndexCommit lastCommit = solrCore.getDeletionPolicy().getLatestCommit(); + + Collection newBackupFiles = newIndexFilesComparedToLastBackup(slice.getName(), lastCommit).stream() + .map(indexFile -> { + Optional backedFile = shardBackupId.getFile(indexFile); + assertTrue(backedFile.isPresent()); + return backedFile.get().uniqueFileName; + }) + .collect(Collectors.toList()); + + lastCommit.getFileNames().forEach( + f -> { + Optional backedFile = shardBackupId.getFile(f); + assertTrue(backedFile.isPresent()); + String uniqueFileName = backedFile.get().uniqueFileName; + + if (newBackupFiles.contains(uniqueFileName)) { + assertTrue(newFilesCopiedOver.contains(repository.resolve(indexDir, uniqueFileName))); + } + + try { + Checksum localChecksum = repository.checksum(dir, f); + Checksum remoteChecksum = backedFile.get().fileChecksum; + assertEquals(localChecksum.checksum, remoteChecksum.checksum); + assertEquals(localChecksum.size, remoteChecksum.size); + } catch (IOException e) { + throw new AssertionError(e); + } + } + ); + + assertEquals("Incremental backup stored more files than needed", lastCommit.getFileNames().size(), shardBackupId.listOriginalFileNames().size()); + } finally { + solrCore.getDirectoryFactory().release(dir); + } + } + } + } + + private Collection newIndexFilesComparedToLastBackup(String shardName, IndexCommit currentCommit) throws IOException { + Collection oldFiles = lastShardCommitToBackupFiles.put(shardName, currentCommit.getFileNames()); + if (oldFiles == null) + oldFiles = new ArrayList<>(); + + List newFiles = new ArrayList<>(currentCommit.getFileNames()); + newFiles.removeAll(oldFiles); + return newFiles; + } + } +} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/package-info.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/package-info.java new file mode 100644 index 00000000000..e0047cef9af --- /dev/null +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Test framework classes for collection APIs + */ +package org.apache.solr.cloud.api.collections; \ No newline at end of file diff --git a/solr/test-framework/src/java/org/apache/solr/core/TrackingBackupRepository.java b/solr/test-framework/src/java/org/apache/solr/core/TrackingBackupRepository.java new file mode 100644 index 00000000000..425ae91ae6c --- /dev/null +++ b/solr/test-framework/src/java/org/apache/solr/core/TrackingBackupRepository.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.core; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.backup.Checksum; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.apache.solr.core.backup.repository.BackupRepositoryFactory; + +public class TrackingBackupRepository implements BackupRepository { + private static final List COPIED_FILES = Collections.synchronizedList(new ArrayList<>()); + + private BackupRepository delegate; + + @Override + public T getConfigProperty(String name) { + return delegate.getConfigProperty(name); + } + + @Override + public URI createURI(String path) { + return delegate.createURI(path); + } + + @Override + public URI resolve(URI baseUri, String... pathComponents) { + return delegate.resolve(baseUri, pathComponents); + } + + @Override + public boolean exists(URI path) throws IOException { + return delegate.exists(path); + } + + @Override + public PathType getPathType(URI path) throws IOException { + return delegate.getPathType(path); + } + + @Override + public String[] listAll(URI path) throws IOException { + return delegate.listAll(path); + } + + @Override + public IndexInput openInput(URI dirPath, String fileName, IOContext ctx) throws IOException { + return delegate.openInput(dirPath, fileName, ctx); + } + + @Override + public OutputStream createOutput(URI path) throws IOException { + return delegate.createOutput(path); + } + + @Override + public void createDirectory(URI path) throws IOException { + delegate.createDirectory(path); + } + + @Override + public void deleteDirectory(URI path) throws IOException { + delegate.deleteDirectory(path); + } + + @Override + public void copyIndexFileFrom(Directory sourceDir, String sourceFileName, URI destDir, String destFileName) throws IOException { + COPIED_FILES.add(delegate.resolve(destDir, destFileName)); + delegate.copyIndexFileFrom(sourceDir, sourceFileName, destDir, destFileName); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + + + @Override + public void delete(URI path, Collection files, boolean ignoreNoSuchFileException) throws IOException { + delegate.delete(path, files, ignoreNoSuchFileException); + } + + @Override + public Checksum checksum(Directory dir, String fileName) throws IOException { + return delegate.checksum(dir, fileName); + } + + @Override + public void init(@SuppressWarnings("rawtypes") NamedList args) { + BackupRepositoryFactory factory = (BackupRepositoryFactory) args.get("factory"); + SolrResourceLoader loader = (SolrResourceLoader) args.get("loader"); + String repoName = (String) args.get("delegateRepoName"); + + this.delegate = factory.newInstance(loader, repoName); + } + + /** + * @return list of files were copied by using {@link #copyFileFrom(Directory, String, URI)} + */ + public static List copiedFiles() { + return new ArrayList<>(COPIED_FILES); + } + + /** + * Clear all tracking data + */ + public static void clear() { + COPIED_FILES.clear(); + } + + @Override + public void copyIndexFileTo(URI sourceRepo, String sourceFileName, Directory dest, String destFileName) throws IOException { + delegate.copyIndexFileTo(sourceRepo, sourceFileName, dest, destFileName); + } +} diff --git a/versions.props b/versions.props index ab51fb6f022..60e191dcb83 100644 --- a/versions.props +++ b/versions.props @@ -8,9 +8,29 @@ com.fasterxml.jackson*:*=2.10.1 com.github.ben-manes.caffeine:caffeine=2.8.4 com.github.virtuald:curvesapi=1.06 com.github.zafarkhaja:java-semver=0.9.0 +com.google.api-client:google-api-client=1.29.2 +com.google.api:api-common=1.8.1 +com.google.api.grpc:proto-google-common-protos=1.16.0 +com.google.api.grpc:proto-google-iam-v1=0.12.0 +com.google.api:gax=1.45.0 +com.google.api:gax-httpjson=0.62.0 +com.google.apis:google-api-services-storage=v1-rev20190426-1.28.0 +com.google.auth:google-auth-library-oauth2-http=0.15.0 +com.google.auth:google-auth-library-credentials=0.15.0 +com.google.code.findbugs:jsr205=3.0.2 +com.google.code.gson:gson=2.7 +com.google.cloud:google-cloud-storage=1.77.0 +com.google.cloud:google-cloud-core=1.77.0 +com.google.cloud:google-cloud-core-http=1.77.0 +com.google.http-client:google-http-client=1.29.2 +com.google.http-client:google-http-client-appengine=1.29.2 +com.google.http-client:google-http-client-jackson2=1.29.2 +com.google.j2objc:j2objc-annotations=1.1 +com.google.oauth-client:google-oauth-client=1.29.2 com.google.errorprone:*=2.4.0 com.google.guava:guava=25.1-jre com.google.protobuf:protobuf-java=3.11.0 +com.google.protobuf:protobuf-java-util=3.11.0 com.google.re2j:re2j=1.2 com.googlecode.juniversalchardet:juniversalchardet=1.0.3 com.googlecode.mp4parser:isoparser=1.1.22 @@ -31,8 +51,11 @@ commons-io:commons-io=2.8.0 commons-logging:commons-logging=1.1.3 de.l3s.boilerpipe:boilerpipe=1.1.0 io.dropwizard.metrics:*=4.1.5 +io.grpc:grpc-context=1.19.0 io.jaegertracing:*=1.1.0 io.netty:*=4.1.50.Final +io.opencensus:opencensus-api=0.21.0 +io.opencensus:opencensus-contrib-http-util=0.21.0 io.opentracing:*=0.33.0 io.prometheus:*=0.2.0 io.sgr:s2-geometry-library-java=1.0.0 @@ -97,6 +120,7 @@ org.ow2.asm:*=7.2 org.rrd4j:rrd4j=3.5 org.slf4j:*=1.7.24 org.tallison:jmatio=1.5 +org.threeten:threetenbp=1.3.3 org.tukaani:xz=1.8 org.xerial.snappy:snappy-java=1.1.7.6 ua.net.nlp:morfologik-ukrainian-search=4.9.1