diff --git a/core/src/main/java/org/fao/geonet/kernel/DataManager.java b/core/src/main/java/org/fao/geonet/kernel/DataManager.java index 4605016efd2..9303b4d5e9c 100644 --- a/core/src/main/java/org/fao/geonet/kernel/DataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/DataManager.java @@ -55,6 +55,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataValidator; import org.fao.geonet.kernel.schema.MetadataSchema; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.IIndexSubmitter; import org.fao.geonet.repository.UserGroupRepository; import org.jdom.Element; import org.slf4j.Logger; @@ -65,7 +66,6 @@ import javax.annotation.CheckForNull; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.io.IOException; import java.nio.file.Path; import java.util.Collection; import java.util.List; @@ -153,19 +153,14 @@ public void batchIndexInThreadPool(ServiceContext context, List metadataIds) metadataIndexer.batchIndexInThreadPool(context, metadataIds); } - @Deprecated - public boolean isIndexing() { - return metadataIndexer.isIndexing(); - } - @Deprecated public void indexMetadata(final List metadataIds) throws Exception { metadataIndexer.indexMetadata(metadataIds); } @Deprecated - public void indexMetadata(final String metadataId, boolean forceRefreshReaders) throws Exception { - metadataIndexer.indexMetadata(metadataId, forceRefreshReaders, IndexingMode.full); + public void indexMetadata(final String metadataId, IIndexSubmitter indexSubmittor) throws Exception { + metadataIndexer.indexMetadata(metadataId, indexSubmittor, IndexingMode.full); } @Deprecated @@ -361,10 +356,10 @@ public String insertMetadata(ServiceContext context, String schema, Element meta @Deprecated public AbstractMetadata insertMetadata(ServiceContext context, AbstractMetadata newMetadata, Element metadataXml, IndexingMode indexingMode, - boolean updateFixedInfo, UpdateDatestamp updateDatestamp, boolean fullRightsForGroup, boolean forceRefreshReaders) + boolean updateFixedInfo, UpdateDatestamp updateDatestamp, boolean fullRightsForGroup, IIndexSubmitter indexSubmittor) throws Exception { return metadataManager.insertMetadata(context, newMetadata, metadataXml, indexingMode, updateFixedInfo, updateDatestamp, - fullRightsForGroup, forceRefreshReaders); + fullRightsForGroup, indexSubmittor); } @Deprecated @@ -578,11 +573,6 @@ public void flush() { metadataManager.flush(); } - @Deprecated - public void forceIndexChanges() throws IOException { - metadataIndexer.forceIndexChanges(); - } - @Deprecated public int batchDeleteMetadataAndUpdateIndex(Specification specification) throws Exception { return metadataIndexer.batchDeleteMetadataAndUpdateIndex(specification); diff --git a/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java b/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java index ac7db5535f3..bd801fdbb78 100644 --- a/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java +++ b/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java @@ -28,14 +28,12 @@ import org.fao.geonet.Util; import org.fao.geonet.constants.Geonet; import org.fao.geonet.domain.User; -import org.fao.geonet.kernel.search.EsSearchManager; +import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmitter; import org.fao.geonet.utils.Log; import org.springframework.transaction.TransactionStatus; -import java.io.IOException; import java.util.List; import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -49,8 +47,6 @@ public final class IndexMetadataTask implements Runnable { private final List _metadataIds; private final TransactionStatus _transactionStatus; private final Set _batchIndex; - private final EsSearchManager searchManager; - private final AtomicInteger indexed; private User _user; /** @@ -61,13 +57,11 @@ public final class IndexMetadataTask implements Runnable { * @param transactionStatus if non-null, wait for the transaction to complete before indexing */ public IndexMetadataTask(@Nonnull ServiceContext context, @Nonnull List metadataIds, Set batchIndex, - @Nullable TransactionStatus transactionStatus, @Nonnull AtomicInteger indexed) { - this.indexed = indexed; + @Nullable TransactionStatus transactionStatus) { this._transactionStatus = transactionStatus; this._context = context; this._metadataIds = metadataIds; this._batchIndex = batchIndex; - this.searchManager = context.getBean(EsSearchManager.class); batchIndex.add(this); @@ -77,7 +71,7 @@ public IndexMetadataTask(@Nonnull ServiceContext context, @Nonnull List metad } public void run() { - try { + try (BatchingIndexSubmitter batchingIndexSubmittor = new BatchingIndexSubmitter(_metadataIds.size())) { _context.setAsThreadLocal(); while (_transactionStatus != null && !_transactionStatus.isCompleted()) { try { @@ -101,13 +95,9 @@ public void run() { DataManager dataManager = _context.getBean(DataManager.class); // servlet up so safe to index all metadata that needs indexing for (Object metadataId : _metadataIds) { - this.indexed.incrementAndGet(); - if (this.indexed.compareAndSet(500, 0)) { - searchManager.forceIndexChanges(); - } try { - dataManager.indexMetadata(metadataId.toString(), false); + dataManager.indexMetadata(metadataId.toString(), batchingIndexSubmittor); } catch (Exception e) { Log.error(Geonet.INDEX_ENGINE, "Error indexing metadata '" + metadataId + "': " + e.getMessage() + "\n" + Util.getStackTrace(e)); @@ -116,7 +106,6 @@ public void run() { if (_user != null && _context.getUserSession().getUserId() == null) { _context.getUserSession().loginAs(_user); } - searchManager.forceIndexChanges(); } finally { _batchIndex.remove(this); } diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java index f7ef10c9c7c..1c95eac1ec5 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java @@ -23,11 +23,11 @@ package org.fao.geonet.kernel.datamanager; -import java.io.IOException; import java.util.List; import org.fao.geonet.domain.AbstractMetadata; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.IIndexSubmitter; import org.jdom.Element; import org.springframework.data.jpa.domain.Specification; @@ -49,11 +49,6 @@ public interface IMetadataIndexer { */ public void init(ServiceContext context, Boolean force) throws Exception; - /** - * Force the index to wait until all changes are processed and the next reader obtained will get the latest data. - */ - void forceIndexChanges() throws IOException; - /** * Remove the records that matches the specification * @@ -77,13 +72,6 @@ public interface IMetadataIndexer { */ void batchIndexInThreadPool(ServiceContext context, List metadataIds); - /** - * Is the platform currently indexing? - * - * @return - */ - boolean isIndexing(); - /** * Index the list of records passed as parameter in order. * @@ -92,7 +80,7 @@ public interface IMetadataIndexer { */ void indexMetadata(List metadataIds) throws Exception; - void indexMetadata(String metadataId, boolean forceRefreshReaders, IndexingMode indexingMode) throws Exception; + void indexMetadata(String metadataId, IIndexSubmitter indexSubmittor, IndexingMode indexingMode) throws Exception; void indexMetadataPrivileges(String uuid, int id) throws Exception; diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataManager.java index 83c4b8d05d9..f9bf29b70e5 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataManager.java @@ -33,6 +33,8 @@ import org.fao.geonet.kernel.EditLib; import org.fao.geonet.kernel.UpdateDatestamp; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.IDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.IIndexSubmitter; import org.fao.geonet.repository.BatchUpdateQuery; import org.fao.geonet.repository.PathSpec; import org.fao.geonet.repository.Updater; @@ -68,9 +70,10 @@ public interface IMetadataManager { * * @param context * @param metadataId + * @param deletionSubmittor * @throws Exception */ - void deleteMetadata(ServiceContext context, String metadataId) throws Exception; + void deleteMetadata(ServiceContext context, String metadataId, IDeletionSubmitter deletionSubmittor) throws Exception; /** * Delete the record with the id metadataId @@ -147,12 +150,12 @@ String insertMetadata(ServiceContext context, String schema, Element metadataXml * @param updateFixedInfo * @param updateDatestamp * @param fullRightsForGroup - * @param forceRefreshReaders + * @param indexSubmittor * @return * @throws Exception */ AbstractMetadata insertMetadata(ServiceContext context, AbstractMetadata newMetadata, Element metadataXml, IndexingMode indexingMode, - boolean updateFixedInfo, UpdateDatestamp updateDatestamp, boolean fullRightsForGroup, boolean forceRefreshReaders) + boolean updateFixedInfo, UpdateDatestamp updateDatestamp, boolean fullRightsForGroup, IIndexSubmitter indexSubmittor) throws Exception; /** diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java index 9b39ce6118b..95225e43b9c 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java @@ -49,6 +49,9 @@ import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexFields; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmitter; +import org.fao.geonet.kernel.search.submission.IIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.*; @@ -69,13 +72,11 @@ import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.interceptor.TransactionAspectSupport; -import java.io.IOException; import java.nio.file.Path; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicInteger; import static org.fao.geonet.resources.Resources.DEFAULT_LOGO_EXTENSION; @@ -147,11 +148,6 @@ public void setMetadataManager(IMetadataManager metadataManager) { Set indexing = new HashSet(); Set batchIndex = ConcurrentHashMap.newKeySet(); - @Override - public void forceIndexChanges() throws IOException { - searchManager.forceIndexChanges(); - } - @Override public int batchDeleteMetadataAndUpdateIndex(Specification specification) throws Exception { @@ -173,29 +169,31 @@ public int batchDeleteMetadataAndUpdateIndex(Specification Integer.toString(input.getId())).collect(Collectors.toList())); // metadataManager.deleteAll(specification); // So delete one by one even if slower - metadataToDelete.forEach(md -> { - try { - // Extract information for RecordDeletedEvent - LinkedHashMap titles = metadataUtils.extractTitles(Integer.toString(md.getId())); - UserSession userSession = ServiceContext.get().getUserSession(); - String xmlBefore = md.getData(); - - store.delResources(ServiceContext.get(), md.getUuid()); - metadataManager.deleteMetadata(ServiceContext.get(), String.valueOf(md.getId())); - - // Trigger RecordDeletedEvent - new RecordDeletedEvent(md.getId(), md.getUuid(), titles, userSession.getUserIdAsInt(), xmlBefore).publish(ApplicationContextHolder.get()); - } catch (Exception e) { - Log.warning(Geonet.DATA_MANAGER, String.format( - - "Error during removal of metadata %s part of batch delete operation. " + - "This error may create a ghost record (ie. not in the index " + - "but still present in the database). " + - "You can reindex the catalogue to see it again. " + - "Error was: %s.", md.getUuid(), e.getMessage())); - e.printStackTrace(); - } - }); + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter(metadataToDelete.size())) { + metadataToDelete.forEach(md -> { + try { + // Extract information for RecordDeletedEvent + LinkedHashMap titles = metadataUtils.extractTitles(Integer.toString(md.getId())); + UserSession userSession = ServiceContext.get().getUserSession(); + String xmlBefore = md.getData(); + + store.delResources(ServiceContext.get(), md.getUuid()); + metadataManager.deleteMetadata(ServiceContext.get(), String.valueOf(md.getId()), submitter); + + // Trigger RecordDeletedEvent + new RecordDeletedEvent(md.getId(), md.getUuid(), titles, userSession.getUserIdAsInt(), xmlBefore).publish(ApplicationContextHolder.get()); + } catch (Exception e) { + Log.warning(Geonet.DATA_MANAGER, String.format( + + "Error during removal of metadata %s part of batch delete operation. " + + "This error may create a ghost record (ie. not in the index " + + "but still present in the database). " + + "You can reindex the catalogue to see it again. " + + "Error was: %s.", md.getUuid(), e.getMessage())); + e.printStackTrace(); + } + }); + } return metadataToDelete.size(); } @@ -268,7 +266,6 @@ public void batchIndexInThreadPool(ServiceContext context, List metadataIds) Log.debug(Geonet.INDEX_ENGINE, "Indexing " + metadataIds.size() + " records."); Log.debug(Geonet.INDEX_ENGINE, metadataIds.toString()); } - AtomicInteger numIndexedTracker = new AtomicInteger(); while (index < metadataIds.size()) { int start = index; int count = Math.min(perThread, metadataIds.size() - start); @@ -285,7 +282,7 @@ public void batchIndexInThreadPool(ServiceContext context, List metadataIds) } // create threads to process this chunk of ids - Runnable worker = new IndexMetadataTask(context, subList, batchIndex, transactionStatus, numIndexedTracker); + Runnable worker = new IndexMetadataTask(context, subList, batchIndex, transactionStatus); executor.execute(worker); index += count; } @@ -293,21 +290,18 @@ public void batchIndexInThreadPool(ServiceContext context, List metadataIds) executor.shutdown(); } - @Override - public boolean isIndexing() { - return searchManager.isIndexing(); - } - @Override public void indexMetadata(final List metadataIds) throws Exception { - for (String metadataId : metadataIds) { - indexMetadata(metadataId, true, IndexingMode.full); + try (BatchingIndexSubmitter indexSubmittor = new BatchingIndexSubmitter(metadataIds.size())) { + for (String metadataId : metadataIds) { + indexMetadata(metadataId, indexSubmittor, IndexingMode.full); + } } } @Override public void indexMetadata(final String metadataId, - final boolean forceRefreshReaders, + final IIndexSubmitter indexSubmittor, final IndexingMode indexingMode) throws Exception { AbstractMetadata fullMd; @@ -388,7 +382,7 @@ public void indexMetadata(final String metadataId, "error", Map.of("record", metadataId, "schema", schema))); searchManager.index(null, md, indexKey, fields, metadataType, - forceRefreshReaders, indexingMode); + indexSubmittor, indexingMode); Log.error(Geonet.DATA_MANAGER, String.format( "Record %s / Schema '%s' is not registered in this catalog. Install it or remove those records. Record is indexed indexing error flag.", metadataId, schema)); @@ -533,7 +527,7 @@ public void indexMetadata(final String metadataId, } searchManager.index(schemaManager.getSchemaDir(schema), md, indexKey, fields, metadataType, - forceRefreshReaders, indexingMode); + indexSubmittor, indexingMode); } } catch (Exception x) { Log.error(Geonet.DATA_MANAGER, "The metadata document index with id=" + metadataId diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java index 7464a267735..405c47e2cc8 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java @@ -50,6 +50,11 @@ import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; import org.fao.geonet.kernel.search.index.BatchOpsMetadataReindexer; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; +import org.fao.geonet.kernel.search.submission.IDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.IIndexSubmitter; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.kernel.setting.SettingInfo; @@ -178,12 +183,9 @@ public void init(ServiceContext context, Boolean force) throws Exception { /** * Refresh index if needed. Can also be called after GeoNetwork startup in * order to rebuild the lucene index - * t * - * @param force Force reindexing all from scratch - * @param asynchronous **/ - public void synchronizeDbWithIndex(ServiceContext context, Boolean force, Boolean asynchronous) throws Exception { + public void synchronizeDbWithIndex(ServiceContext context) throws Exception { // get lastchangedate of all metadata in index Map docs = searchManager.getDocsChangeDate(); @@ -224,7 +226,7 @@ public void synchronizeDbWithIndex(ServiceContext context, Boolean force, Boolea LOGGER_DATA_MANAGER.debug("- idxLastChange: {}", idxLastChange); // date in index contains 't', date in DBMS contains 'T' - if (force || !idxLastChange.equalsIgnoreCase(lastChange)) { + if (!idxLastChange.equalsIgnoreCase(lastChange)) { LOGGER_DATA_MANAGER.debug("- will be indexed"); toIndex.add(id); } @@ -238,25 +240,23 @@ public void synchronizeDbWithIndex(ServiceContext context, Boolean force, Boolea // if anything to index then schedule it to be done after servlet is // up so that any links to local fragments are resolvable - if (toIndex.size() > 0) { - if (asynchronous) { - Set integerList = toIndex.stream().map(Integer::parseInt).collect(Collectors.toSet()); - new BatchOpsMetadataReindexer( - context.getBean(DataManager.class), - integerList).process(settingManager.getSiteId(), false); - } else { - metadataIndexer.batchIndexInThreadPool(context, toIndex); - } + if (!toIndex.isEmpty()) { + Set integerList = toIndex.stream().map(Integer::parseInt).collect(Collectors.toSet()); + new BatchOpsMetadataReindexer( + context.getBean(DataManager.class), + integerList).process(settingManager.getSiteId(), false); } - if (docs.size() > 0) { // anything left? + if (!docs.isEmpty()) { // anything left? LOGGER_DATA_MANAGER.debug("INDEX HAS RECORDS THAT ARE NOT IN DB:"); } // remove from index metadata not in DBMS - for (String id : docs.keySet()) { - getSearchManager().delete(String.format("+id:%s", id)); - LOGGER_DATA_MANAGER.debug("- removed record ({}) from index", id); + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter()) { + for (String id : docs.keySet()) { + LOGGER_DATA_MANAGER.debug("- removing record ({}) from index", id); + getSearchManager().deleteByQuery(String.format("+id:%s", id), submitter); + } } } @@ -335,18 +335,17 @@ private XmlSerializer getXmlSerializer() { * Removes a metadata. */ @Override - public void deleteMetadata(ServiceContext context, String metadataId) throws Exception { + public void deleteMetadata(ServiceContext context, String metadataId, IDeletionSubmitter submitter) throws Exception { AbstractMetadata findOne = metadataUtils.findOne(metadataId); if (findOne != null) { boolean isMetadata = findOne.getDataInfo().getType() == MetadataType.METADATA; deleteMetadataFromDB(context, metadataId); + // --- update search criteria + getSearchManager().deleteByUuid(findOne.getUuid(), submitter); + } else { + getSearchManager().deleteByQuery(String.format("+id:%s", metadataId), submitter); } - - // --- update search criteria - getSearchManager().delete(String.format("+id:%s", metadataId)); - // _entityManager.flush(); - // _entityManager.clear(); } /** @@ -375,7 +374,7 @@ public void purgeMetadata(ServiceContext context, String metadataId, boolean wit RecordDeletedEvent recordDeletedEvent = new RecordDeletedEvent( metadata.getId(), metadata.getUuid(), new LinkedHashMap<>(), context.getUserSession().getUserIdAsInt(), metadata.getData()); - deleteMetadata(context, metadataId); + deleteMetadata(context, metadataId, DirectDeletionSubmitter.INSTANCE); recordDeletedEvent.publish(ApplicationContextHolder.get()); } @@ -387,7 +386,7 @@ public void purgeMetadata(ServiceContext context, String metadataId, boolean wit @Override public void deleteMetadataGroup(ServiceContext context, String metadataId) throws Exception { deleteMetadataFromDB(context, metadataId); - getSearchManager().delete(String.format("+id:%s", metadataId)); + getSearchManager().deleteByQuery(String.format("+id:%s", metadataId), DirectDeletionSubmitter.INSTANCE); } /** @@ -464,7 +463,7 @@ public boolean apply(@Nullable MetadataCategory input) { newMetadata.getMetadataCategories().addAll(filteredCategories); int finalId = insertMetadata(context, newMetadata, xml, IndexingMode.full, true, UpdateDatestamp.YES, - fullRightsForGroup, true).getId(); + fullRightsForGroup, DirectIndexSubmitter.INSTANCE).getId(); return String.valueOf(finalId); } @@ -569,7 +568,7 @@ public String insertMetadata(ServiceContext context, String schema, Element meta boolean fullRightsForGroup = false; int finalId = insertMetadata(context, newMetadata, metadataXml, indexingMode, ufo, UpdateDatestamp.NO, - fullRightsForGroup, true).getId(); + fullRightsForGroup, DirectIndexSubmitter.INSTANCE).getId(); return String.valueOf(finalId); } @@ -577,7 +576,7 @@ public String insertMetadata(ServiceContext context, String schema, Element meta @Override public AbstractMetadata insertMetadata(ServiceContext context, AbstractMetadata newMetadata, Element metadataXml, IndexingMode indexingMode, boolean updateFixedInfo, UpdateDatestamp updateDatestamp, - boolean fullRightsForGroup, boolean forceRefreshReaders) throws Exception { + boolean fullRightsForGroup, IIndexSubmitter indexSubmittor) throws Exception { final String schema = newMetadata.getDataInfo().getSchemaId(); // Check if the schema is allowed by settings @@ -616,7 +615,7 @@ public AbstractMetadata insertMetadata(ServiceContext context, AbstractMetadata metadataOperations.copyDefaultPrivForGroup(context, stringId, groupId, fullRightsForGroup); if (indexingMode != IndexingMode.none) { - metadataIndexer.indexMetadata(stringId, forceRefreshReaders, indexingMode); + metadataIndexer.indexMetadata(stringId, indexSubmittor, indexingMode); } return savedMetadata; @@ -787,9 +786,9 @@ public synchronized AbstractMetadata updateMetadata(final ServiceContext context if (indexingMode != IndexingMode.none) { // Delete old record if UUID changed if (uuidBeforeUfo != null && !uuidBeforeUfo.equals(uuid)) { - getSearchManager().delete(String.format("+uuid:\"%s\"", uuidBeforeUfo)); + getSearchManager().deleteByUuid(uuidBeforeUfo, DirectDeletionSubmitter.INSTANCE); } - metadataIndexer.indexMetadata(metadataId, true, indexingMode); + metadataIndexer.indexMetadata(metadataId, DirectIndexSubmitter.INSTANCE, indexingMode); } } diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataStatus.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataStatus.java index cf5f7fc972d..27aaec71bf9 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataStatus.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataStatus.java @@ -38,6 +38,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataStatus; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.repository.GroupRepository; import org.fao.geonet.repository.MetadataStatusRepository; @@ -155,7 +156,7 @@ public String getCurrentStatus(int metadataId) throws Exception { public MetadataStatus setStatus(ServiceContext context, int id, int status, ISODate changeDate, String changeMessage) throws Exception { MetadataStatus statusObject = setStatusExt(context, id, status, changeDate, changeMessage); - metadataIndexer.indexMetadata(Integer.toString(id), true, IndexingMode.full); + metadataIndexer.indexMetadata(Integer.toString(id), DirectIndexSubmitter.INSTANCE, IndexingMode.full); return statusObject; } @@ -163,7 +164,7 @@ public MetadataStatus setStatus(ServiceContext context, int id, int status, ISOD public MetadataStatus setStatusExt(MetadataStatus metatatStatus, boolean updateIndex) throws Exception { metadataStatusRepository.save(metatatStatus); if (updateIndex) { - metadataIndexer.indexMetadata(metatatStatus.getMetadataId() + "", true, IndexingMode.full); + metadataIndexer.indexMetadata(metatatStatus.getMetadataId() + "", DirectIndexSubmitter.INSTANCE, IndexingMode.full); } return metatatStatus; } @@ -250,7 +251,7 @@ public void changeCurrentStatus(Integer userId, Integer metadataId, Integer newS metatatStatus.setTitles(metadataUtils.extractTitles(Integer.toString(metadataId))); metadataStatusRepository.save(metatatStatus); - metadataIndexer.indexMetadata(metadataId + "", true, IndexingMode.full); + metadataIndexer.indexMetadata(metadataId + "", DirectIndexSubmitter.INSTANCE, IndexingMode.full); } // Utility to verify workflow status transitions diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataUtils.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataUtils.java index bbcbe009e6d..f5de1631759 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataUtils.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataUtils.java @@ -43,6 +43,7 @@ import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; import org.fao.geonet.kernel.search.index.IndexingList; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.lib.Lib; @@ -512,7 +513,7 @@ public String getNewVersion(String id) { @Override public void setTemplate(final int id, final MetadataType type, final String title) throws Exception { setTemplateExt(id, type); - metadataIndexer.indexMetadata(Integer.toString(id), true, IndexingMode.full); + metadataIndexer.indexMetadata(Integer.toString(id), DirectIndexSubmitter.INSTANCE, IndexingMode.full); } @Override @@ -526,7 +527,7 @@ public void setTemplateExt(final int id, final MetadataType metadataType) throws @Override public void setHarvested(int id, String harvestUuid) throws Exception { setHarvestedExt(id, harvestUuid); - metadataIndexer.indexMetadata(Integer.toString(id), true, IndexingMode.full); + metadataIndexer.indexMetadata(Integer.toString(id), DirectIndexSubmitter.INSTANCE, IndexingMode.full); } /** @@ -767,7 +768,7 @@ private void transformMd(ServiceContext context, String metadataId, Element md, getXmlSerializer().update(metadataId, md, changeDate, true, uuid, context); if (indexAfterChange) { - metadataIndexer.indexMetadata(metadataId, true, IndexingMode.full); + metadataIndexer.indexMetadata(metadataId, DirectIndexSubmitter.INSTANCE, IndexingMode.full); } } diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataManager.java index 07db4799fe2..2850a1ed6b9 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataManager.java @@ -32,6 +32,7 @@ import org.fao.geonet.domain.MetadataType; import org.fao.geonet.kernel.datamanager.IMetadataManager; import org.fao.geonet.kernel.datamanager.base.BaseMetadataManager; +import org.fao.geonet.kernel.search.submission.IDeletionSubmitter; import org.fao.geonet.repository.MetadataDraftRepository; import org.fao.geonet.repository.PathSpec; import org.fao.geonet.repository.Updater; @@ -69,7 +70,7 @@ public void init(ServiceContext context, Boolean force) throws Exception { * Removes a metadata. */ @Override - public void deleteMetadata(ServiceContext context, String metadataId) throws Exception { + public void deleteMetadata(ServiceContext context, String metadataId, IDeletionSubmitter submitter) throws Exception { AbstractMetadata findOne = metadataUtils.findOne(metadataId); if (findOne != null) { @@ -86,12 +87,10 @@ public void deleteMetadata(ServiceContext context, String metadataId) throws Exc } deleteMetadataFromDB(context, metadataId); + getSearchManager().deleteByUuid(findOne.getUuid(), submitter); + } else { + getSearchManager().deleteByQuery(String.format("+id:%s", metadataId), submitter); } - - // --- update search criteria - getSearchManager().delete(String.format("+id:%s", metadataId)); - // _entityManager.flush(); - // _entityManager.clear(); } /** diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java index 39b163ded5b..0a9461ab8ae 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java @@ -41,6 +41,8 @@ import org.fao.geonet.kernel.metadata.StatusActionsFactory; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.*; import org.fao.geonet.repository.specification.MetadataFileUploadSpecs; @@ -521,7 +523,7 @@ protected String createDraft(ServiceContext context, String templateId, String g try { newMetadata = (MetadataDraft) metadataManager.insertMetadata(context, newMetadata, xml, IndexingMode.full, true, - UpdateDatestamp.YES, false, true); + UpdateDatestamp.YES, false, DirectIndexSubmitter.INSTANCE); Integer finalId = newMetadata.getId(); @@ -651,7 +653,7 @@ public void cancelEditingSession(ServiceContext context, String id) throws Excep // --- remove metadata xmlSerializer.delete(id, ServiceContext.get()); - searchManager.delete(String.format("+id:%s", id)); + searchManager.deleteByQuery(String.format("+id:%s", id), DirectDeletionSubmitter.INSTANCE); // Unset METADATA_EDITING_CREATED_DRAFT flag context.getUserSession().removeProperty(Geonet.Session.METADATA_EDITING_CREATED_DRAFT); diff --git a/core/src/main/java/org/fao/geonet/kernel/mef/Importer.java b/core/src/main/java/org/fao/geonet/kernel/mef/Importer.java index 1200ff5e913..7e15f9435d6 100644 --- a/core/src/main/java/org/fao/geonet/kernel/mef/Importer.java +++ b/core/src/main/java/org/fao/geonet/kernel/mef/Importer.java @@ -45,6 +45,8 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.datamanager.IMetadataValidator; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.*; @@ -444,7 +446,7 @@ public void apply(@Nonnull final Metadata metadata) { } } - dm.indexMetadata(metadataIdMap.get(index), true); + dm.indexMetadata(metadataIdMap.get(index), DirectIndexSubmitter.INSTANCE); } // -------------------------------------------------------------------- @@ -571,7 +573,7 @@ public static void importRecord(String uuid, MEFLib.UuidAction uuidAction, List< if (Log.isDebugEnabled(Geonet.MEF)) { Log.debug(Geonet.MEF, "Deleting existing metadata with UUID : " + uuid); } - metadataManager.deleteMetadata(context, dm.getMetadataId(uuid)); + metadataManager.deleteMetadata(context, dm.getMetadataId(uuid), DirectDeletionSubmitter.INSTANCE); metadataManager.flush(); } else { throw new UnAuthorizedException("User has no privilege to replace existing metadata", null); diff --git a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java index 978ab63a750..355cd7f86f2 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java @@ -24,9 +24,11 @@ package org.fao.geonet.kernel.search; import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch._types.Refresh; import co.elastic.clients.elasticsearch._types.SortOptions; import co.elastic.clients.elasticsearch.core.*; import co.elastic.clients.elasticsearch.core.bulk.BulkOperation; +import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem; import co.elastic.clients.elasticsearch.core.bulk.UpdateOperation; import co.elastic.clients.elasticsearch.core.search.Hit; import co.elastic.clients.elasticsearch.indices.ExistsRequest; @@ -53,6 +55,10 @@ import org.fao.geonet.kernel.datamanager.IMetadataIndexer; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.index.OverviewIndexFieldUpdater; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.IDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmitter; +import org.fao.geonet.kernel.search.submission.IIndexSubmitter; import org.fao.geonet.kernel.setting.SettingInfo; import org.fao.geonet.repository.SourceRepository; import org.fao.geonet.repository.specification.MetadataSpecs; @@ -173,11 +179,6 @@ public void setIndexType(String indexType) { @Autowired private OverviewIndexFieldUpdater overviewFieldUpdater; - private int commitInterval = 200; - - // public for test, to be private or protected - public Map listOfDocumentsToIndex = - Collections.synchronizedMap(new HashMap<>()); private Map indexList; private Path getXSLTForIndexing(Path schemaDir, MetadataType metadataType) { @@ -383,6 +384,7 @@ public BulkResponse updateFields(String id, Map fieldMap, Set b.index(defaultIndex) .operations(bulkOperationList) + .refresh(Refresh.True) ); return client.getClient().bulk(bulkRequest); @@ -397,7 +399,7 @@ public void updateFieldsAsynch(String id, Map fields) { .doc(fields) ); - client.getAsynchClient() + client.getAsyncClient() .update(updateRequest, ObjectNode.class) .whenComplete((response, exception) -> { if (exception != null) { @@ -427,7 +429,7 @@ public void updateFieldAsynch(String id, String field, Object value) { public void index(Path schemaDir, Element metadata, String id, Multimap dbFields, MetadataType metadataType, - boolean forceRefreshReaders, + IIndexSubmitter indexSubmittor, IndexingMode indexingMode) throws Exception { Element docs = new Element("doc"); @@ -453,36 +455,40 @@ public void index(Path schemaDir, Element metadata, String id, String jsonDocument = mapper.writeValueAsString(doc); - if (forceRefreshReaders) { - Map document = new HashMap<>(); - document.put(id, jsonDocument); - final BulkResponse bulkItemResponses = client.bulkRequest(defaultIndex, document); - checkIndexResponse(bulkItemResponses, document); - overviewFieldUpdater.process(id); - } else { - listOfDocumentsToIndex.put(id, jsonDocument); - if (listOfDocumentsToIndex.size() == commitInterval) { - sendDocumentsToIndex(); + indexSubmittor.submitToIndex(id, jsonDocument, this); + } + + public void handleIndexResponse(BulkResponse bulkResponse, Map documents) throws IOException { + try { + checkIndexResponse(bulkResponse, documents); + } finally { + for (String uuid : documents.keySet()) { + overviewFieldUpdater.process(uuid); } } } - private void sendDocumentsToIndex() { - Map documents = new HashMap<>(listOfDocumentsToIndex); - listOfDocumentsToIndex.clear(); - if (!documents.isEmpty()) { - try { - final BulkResponse bulkItemResponses = client - .bulkRequest(defaultIndex, documents); - checkIndexResponse(bulkItemResponses, documents); - } catch (Exception e) { - LOGGER.error( - "An error occurred while indexing {} documents in current indexing list. Error is {}.", - listOfDocumentsToIndex.size(), e.getMessage()); - } finally { - // TODO: Trigger this async ? - documents.keySet().forEach(uuid -> overviewFieldUpdater.process(uuid)); + public void handleDeletionResponse(BulkResponse bulkResponse, List documents) { + if (bulkResponse.errors()) { + StringBuilder builder = new StringBuilder(); + for (BulkResponseItem item : bulkResponse.items()) { + if (item.error() != null) { + builder.append("Failed to delete document ").append(item.id()).append(" from index: ").append(item.error()).append("\n"); + } } + throw new RuntimeException("Some documents could not be deleted from the index!\n" + builder.toString()); + } + } + + public void handleDeletionResponse(DeleteByQueryResponse deleteByQueryResponse, String query) { + if (!deleteByQueryResponse.failures().isEmpty()) { + StringBuilder stringBuilder = new StringBuilder(); + + deleteByQueryResponse.failures().forEach(f -> stringBuilder.append(f.toString())); + + throw new RuntimeException(String.format( + "Error during removal of query %s. Errors are '%s'.", query, stringBuilder.toString() + )); } } @@ -537,7 +543,8 @@ private void checkIndexResponse(BulkResponse bulkItemResponses, }); if (!listErrorOfDocumentsToIndex.isEmpty()) { - BulkResponse response = client.bulkRequest(defaultIndex, listErrorOfDocumentsToIndex); + BulkRequest bulkRequest = client.buildIndexBulkRequest(defaultIndex, listErrorOfDocumentsToIndex); + BulkResponse response = client.getClient().bulk(bulkRequest); if (response.errors()) { LOGGER.error("Failed to save error documents {}.", Arrays.toString(errorDocumentIds.toArray())); @@ -546,9 +553,9 @@ private void checkIndexResponse(BulkResponse bulkItemResponses, } } - private static ImmutableSet booleanFields; - private static ImmutableSet arrayFields; - private static ImmutableSet booleanValues; + private static final ImmutableSet booleanFields; + private static final ImmutableSet arrayFields; + private static final ImmutableSet booleanValues; static { arrayFields = ImmutableSet.builder() @@ -723,11 +730,6 @@ private String parseBoolean(String value) { return String.valueOf(booleanValues.contains(value)); } - @Override - public void forceIndexChanges() { - sendDocumentsToIndex(); - } - @Override public boolean rebuildIndex(ServiceContext context, boolean reset, String bucket) throws Exception { @@ -744,9 +746,7 @@ public boolean rebuildIndex(ServiceContext context, SelectionManager sm = SelectionManager.getManager(session); synchronized (sm.getSelection(bucket)) { - for (Iterator iter = sm.getSelection(bucket).iterator(); - iter.hasNext(); ) { - String uuid = iter.next(); + for (String uuid : sm.getSelection(bucket)) { for (AbstractMetadata metadata : metadataRepository.findAllByUuid(uuid)) { String indexKey = uuid; if (metadata instanceof MetadataDraft) { @@ -761,8 +761,10 @@ public boolean rebuildIndex(ServiceContext context, } } } - for (String id : listOfIdsToIndex) { - metadataIndexer.indexMetadata(id, false, IndexingMode.full); + try (BatchingIndexSubmitter indexSubmittor = new BatchingIndexSubmitter(listOfIdsToIndex.size())) { + for (String id : listOfIdsToIndex) { + metadataIndexer.indexMetadata(id, indexSubmittor, IndexingMode.full); + } } } else { final Specification metadataSpec = @@ -771,11 +773,12 @@ public boolean rebuildIndex(ServiceContext context, final List metadataIds = metadataRepository.findAllIdsBy( Specification.where(metadataSpec) ); - for (Integer id : metadataIds) { - metadataIndexer.indexMetadata(id + "", false, IndexingMode.full); + try (BatchingIndexSubmitter indexSubmittor = new BatchingIndexSubmitter(metadataIds.size())) { + for (Integer id : metadataIds) { + metadataIndexer.indexMetadata(id + "", indexSubmittor, IndexingMode.full); + } } } - sendDocumentsToIndex(); return true; } @@ -820,7 +823,7 @@ public Map getFieldsValues(String id, Set fields, String public void clearIndex() throws Exception { - client.deleteByQuery(defaultIndex, "*:*"); + deleteByQuery("*:*", DirectDeletionSubmitter.INSTANCE); } static ImmutableSet docsChangeIncludedFields; @@ -878,19 +881,13 @@ public ISODate getDocChangeDate(String mdId) throws Exception { } @Override - public void delete(String txt) throws Exception { - client.deleteByQuery(defaultIndex, txt); + public void deleteByQuery(String query, IDeletionSubmitter submitter) throws Exception { + submitter.submitQueryToIndex(query, this); } @Override - public void delete(List metadataIds) throws Exception { - metadataIds.forEach(metadataId -> { - try { - this.delete(String.format("+id:%d", metadataId)); - } catch (Exception e) { - e.printStackTrace(); - } - }); + public void deleteByUuid(String uuid, IDeletionSubmitter submitter) throws Exception { + submitter.submitUUIDToIndex(uuid, this); } public long getNumDocs(String query) throws Exception { @@ -929,10 +926,6 @@ public static String analyzeField(String analyzer, fieldValue); } - public boolean isIndexing() { - return listOfDocumentsToIndex.size() > 0; - } - public boolean isIndexWritable(String indexName) throws IOException, ElasticsearchException { String indexBlockRead = "index.blocks.read_only_allow_delete"; diff --git a/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java index f7d4959953b..53ccd3caed1 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java @@ -28,6 +28,8 @@ import jeeves.server.context.ServiceContext; import org.fao.geonet.domain.ISODate; import org.fao.geonet.domain.MetadataType; +import org.fao.geonet.kernel.search.submission.IDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.IIndexSubmitter; import org.jdom.Element; import java.io.IOException; @@ -47,19 +49,12 @@ public interface ISearchManager { /** * Indexes a metadata record. * - * @param forceRefreshReaders if true then block all searches until they can obtain a up-to-date - * reader + * @param indexSubmittor The submitter to use */ void index(Path schemaDir, Element metadata, String id, Multimap moreFields, - MetadataType metadataType, boolean forceRefreshReaders, IndexingMode indexingMode) + MetadataType metadataType, IIndexSubmitter indexSubmittor, IndexingMode indexingMode) throws Exception; - /** - * Force the index to wait until all changes are processed and the next reader obtained will get - * the latest data. - */ - void forceIndexChanges() throws IOException; - /** * Rebuilds the Lucene index. If xlink or from selection parameters are defined, reindex a @@ -76,14 +71,14 @@ boolean rebuildIndex(ServiceContext context, ISODate getDocChangeDate(String mdId) throws Exception; /** - * deletes a document. + * deletes a document by a query. */ - void delete(String txt) throws Exception; + void deleteByQuery(String query, IDeletionSubmitter submitter) throws Exception; /** - * deletes a list of documents. + * deletes a document by its uuid. */ - void delete(List metadataIds) throws Exception; + void deleteByUuid(String uuid, IDeletionSubmitter submitter) throws Exception; boolean isIndexWritable(String indexName) throws IOException, ElasticsearchException; } diff --git a/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java b/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java index 22ad1941980..be36b13e66e 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java @@ -45,12 +45,11 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.RemovalNotification; import com.google.common.collect.Lists; -import com.google.common.util.concurrent.MoreExecutors; import org.fao.geonet.ApplicationContextHolder; import org.fao.geonet.constants.Geonet; import org.fao.geonet.kernel.DataManager; import org.fao.geonet.kernel.MetadataIndexerProcessor; -import org.fao.geonet.kernel.search.EsSearchManager; +import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmitter; import org.fao.geonet.util.ThreadUtils; import org.fao.geonet.utils.Log; import org.springframework.jmx.export.MBeanExporter; @@ -58,8 +57,10 @@ import org.springframework.jmx.export.annotation.ManagedResource; import javax.management.ObjectName; +import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.WeakHashMap; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -74,11 +75,17 @@ @ManagedResource() public class BatchOpsMetadataReindexer extends MetadataIndexerProcessor implements Runnable { - private static final JmxRemovalListener removalListener = new JmxRemovalListener(); + private static final JmxRemovalListener REMOVAL_LISTENER = new JmxRemovalListener(); private static final Cache PROBE_CACHE = CacheBuilder.newBuilder() .expireAfterWrite(1, TimeUnit.MINUTES) - .removalListener(removalListener) + .removalListener(REMOVAL_LISTENER) .build(); + private static final Set> RUNNING_INDEXERS = Collections.synchronizedSet(Collections.newSetFromMap(new WeakHashMap<>())); + + public static boolean isIndexing() { + RUNNING_INDEXERS.removeIf(CompletableFuture::isDone); + return !RUNNING_INDEXERS.isEmpty(); + } private final Set metadata; private ExecutorService executor = null; @@ -94,7 +101,7 @@ public BatchOpsMetadataReindexer(DataManager dm, Set metadata) { this.metadata = metadata; this.toProcessCount = metadata.size(); exporter = ApplicationContextHolder.get().getBean(MBeanExporter.class); - removalListener.setExporter(exporter); + REMOVAL_LISTENER.setExporter(exporter); } @ManagedAttribute @@ -129,15 +136,9 @@ public String wrapAsyncProcess(String catalogueId, boolean runInCurrentThread) t return processAsync(runInCurrentThread); } - private String processAsync(boolean runInCurrentThread) throws Exception { + private String processAsync(boolean runInCurrentThread) { int threadCount = ThreadUtils.getNumberOfThreads(); - if (runInCurrentThread) { - executor = MoreExecutors.newDirectExecutorService(); - } else { - executor = Executors.newFixedThreadPool(threadCount); - } - int[] ids = metadata.stream().mapToInt(i -> i).toArray(); int perThread; @@ -161,20 +162,38 @@ private String processAsync(boolean runInCurrentThread) throws Exception { index += count; } - List submitList = Lists.newArrayList(); - for (BatchOpsCallable job : jobs) { - CompletableFuture completed = CompletableFuture.runAsync(job, executor); - submitList.add(completed); - } + if (runInCurrentThread) { + allCompleted = new CompletableFuture<>(); + try { + RUNNING_INDEXERS.add(allCompleted); + for (BatchOpsCallable job : jobs) { + job.run(); + } + } + finally { + allCompleted.complete(null); + } + } else { + List> submitList = Lists.newArrayList(); + executor = Executors.newFixedThreadPool(threadCount); - allCompleted = CompletableFuture.allOf(submitList.toArray(new CompletableFuture[submitList.size()])); + for (BatchOpsCallable job : jobs) { + CompletableFuture completed = CompletableFuture.runAsync(job, executor); + submitList.add(completed); + } + + allCompleted = CompletableFuture.allOf(submitList.toArray(new CompletableFuture[0])); + RUNNING_INDEXERS.add(allCompleted); + } allCompleted.thenRun(this); return probeName.toString(); } @Override public void run() { - executor.shutdown(); + if (executor != null) { + executor.shutdown(); + } PROBE_CACHE.cleanUp(); PROBE_CACHE.put(probeName, probeName); } @@ -208,20 +227,21 @@ public void run() { "Indexing range [%d-%d]/%d by threads %s.", beginIndex, beginIndex + count, ids.length, Thread.currentThread().getId())); long start = System.currentTimeMillis(); - for (int i = beginIndex; i < beginIndex + count; i++) { - try { - dm.indexMetadata(ids[i] + "", false); - processed.incrementAndGet(); - } catch (Exception e) { - inError.incrementAndGet(); + try (BatchingIndexSubmitter batchingIndexSubmittor = new BatchingIndexSubmitter(count)) { + for (int i = beginIndex; i < beginIndex + count; i++) { + try { + dm.indexMetadata(ids[i] + "", batchingIndexSubmittor); + processed.incrementAndGet(); + } catch (Exception e) { + inError.incrementAndGet(); + } } + Log.warning(Geonet.INDEX_ENGINE, String.format( + "Indexing range [%d-%d]/%d completed in %dms by threads %s.", + beginIndex, beginIndex + count, ids.length, + System.currentTimeMillis() - start, + Thread.currentThread().getId())); } - Log.warning(Geonet.INDEX_ENGINE, String.format( - "Indexing range [%d-%d]/%d completed in %dms by threads %s.", - beginIndex, beginIndex + count, ids.length, - System.currentTimeMillis() - start, - Thread.currentThread().getId())); - ApplicationContextHolder.get().getBean(EsSearchManager.class).forceIndexChanges(); } } } diff --git a/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java b/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java index dfb4eb305ec..8fd22e30ae1 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java @@ -29,7 +29,7 @@ import org.fao.geonet.ApplicationContextHolder; import org.fao.geonet.constants.Geonet; import org.fao.geonet.kernel.DataManager; -import org.fao.geonet.kernel.search.EsSearchManager; +import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmitter; import org.fao.geonet.utils.Log; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; @@ -37,7 +37,6 @@ import org.springframework.context.ConfigurableApplicationContext; import org.springframework.scheduling.quartz.QuartzJobBean; -import java.io.IOException; import java.util.Date; import java.util.Set; @@ -63,21 +62,25 @@ private void indexRecords() { ApplicationContextHolder.set(applicationContext); IndexingList list = applicationContext.getBean(IndexingList.class); Set metadataIdentifiers = list.getIdentifiers(); - if (metadataIdentifiers.size() > 0) { - if (Log.isDebugEnabled(Geonet.INDEX_ENGINE)) { - Log.debug(Geonet.INDEX_ENGINE, "Indexing task / List of records to index: " - + metadataIdentifiers.toString() + "."); - } + if (metadataIdentifiers.isEmpty()) { + return; + } + + + if (Log.isDebugEnabled(Geonet.INDEX_ENGINE)) { + Log.debug(Geonet.INDEX_ENGINE, "Indexing task / List of records to index: " + + metadataIdentifiers.toString() + "."); + } + try (BatchingIndexSubmitter batchingIndexSubmittor = new BatchingIndexSubmitter(metadataIdentifiers.size())) { for (Integer metadataIdentifier : metadataIdentifiers) { try { - _dataManager.indexMetadata(String.valueOf(metadataIdentifier), false); + _dataManager.indexMetadata(String.valueOf(metadataIdentifier), batchingIndexSubmittor); } catch (Exception e) { Log.error(Geonet.INDEX_ENGINE, "Indexing task / An error happens indexing the metadata " + metadataIdentifier + ". Error: " + e.getMessage(), e); } } - this.applicationContext.getBean(EsSearchManager.class).forceIndexChanges(); } } diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectDeletionSubmitter.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectDeletionSubmitter.java new file mode 100644 index 00000000000..c8dc445ede4 --- /dev/null +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectDeletionSubmitter.java @@ -0,0 +1,39 @@ +package org.fao.geonet.kernel.search.submission; + +import co.elastic.clients.elasticsearch.core.BulkRequest; +import co.elastic.clients.elasticsearch.core.BulkResponse; +import co.elastic.clients.elasticsearch.core.DeleteByQueryRequest; +import co.elastic.clients.elasticsearch.core.DeleteByQueryResponse; +import org.fao.geonet.index.es.EsRestClient; +import org.fao.geonet.kernel.search.EsSearchManager; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class DirectDeletionSubmitter implements IDeletionSubmitter { + public static final DirectDeletionSubmitter INSTANCE = new DirectDeletionSubmitter(); + + private DirectDeletionSubmitter() {} + + @Override + public void submitUUIDToIndex(String uuid, EsSearchManager searchManager) throws IOException { + EsRestClient restClient = searchManager.getClient(); + List documents = Collections.singletonList(uuid); + + BulkRequest bulkRequest = restClient.buildDeleteBulkRequest(searchManager.getDefaultIndex(), documents); + final BulkResponse bulkItemResponses = restClient.getClient().bulk(bulkRequest); + + searchManager.handleDeletionResponse(bulkItemResponses, documents); + } + + @Override + public void submitQueryToIndex(String query, EsSearchManager searchManager) throws IOException { + EsRestClient restClient = searchManager.getClient(); + + DeleteByQueryRequest deleteByQueryRequest = restClient.buildDeleteByQuery(searchManager.getDefaultIndex(), query); + final DeleteByQueryResponse deleteByQueryResponse = restClient.getClient().deleteByQuery(deleteByQueryRequest); + + searchManager.handleDeletionResponse(deleteByQueryResponse, query); + } +} diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectIndexSubmitter.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectIndexSubmitter.java new file mode 100644 index 00000000000..9ac32a00668 --- /dev/null +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectIndexSubmitter.java @@ -0,0 +1,30 @@ +package org.fao.geonet.kernel.search.submission; + +import co.elastic.clients.elasticsearch.core.BulkRequest; +import co.elastic.clients.elasticsearch.core.BulkResponse; +import org.fao.geonet.index.es.EsRestClient; +import org.fao.geonet.kernel.search.EsSearchManager; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +/** + * An index submitter that directly and synchronously transmits new documents to the index + */ +public class DirectIndexSubmitter implements IIndexSubmitter { + public static final DirectIndexSubmitter INSTANCE = new DirectIndexSubmitter(); + + private DirectIndexSubmitter() {} + + @Override + public void submitToIndex(String id, String jsonDocument, EsSearchManager searchManager) throws IOException { + EsRestClient restClient = searchManager.getClient(); + Map documents = Collections.singletonMap(id, jsonDocument); + + BulkRequest bulkRequest = restClient.buildIndexBulkRequest(searchManager.getDefaultIndex(), documents); + final BulkResponse bulkItemResponses = restClient.getClient().bulk(bulkRequest); + + searchManager.handleIndexResponse(bulkItemResponses, documents); + } +} diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/IDeletionSubmitter.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/IDeletionSubmitter.java new file mode 100644 index 00000000000..745a8bb706e --- /dev/null +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/IDeletionSubmitter.java @@ -0,0 +1,12 @@ +package org.fao.geonet.kernel.search.submission; + +import org.fao.geonet.kernel.search.EsSearchManager; + +import java.io.IOException; + +public interface IDeletionSubmitter { + + void submitUUIDToIndex(String uuid, EsSearchManager searchManager) throws IOException; + + void submitQueryToIndex(String query, EsSearchManager searchManager) throws IOException; +} diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/IIndexSubmitter.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/IIndexSubmitter.java new file mode 100644 index 00000000000..11fd9e26f8e --- /dev/null +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/IIndexSubmitter.java @@ -0,0 +1,10 @@ +package org.fao.geonet.kernel.search.submission; + +import org.fao.geonet.kernel.search.EsSearchManager; + +import java.io.IOException; + +public interface IIndexSubmitter { + + void submitToIndex(String id, String jsonDocument, EsSearchManager searchManager) throws IOException; +} diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingDeletionSubmitter.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingDeletionSubmitter.java new file mode 100644 index 00000000000..3fe31ffda76 --- /dev/null +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingDeletionSubmitter.java @@ -0,0 +1,77 @@ +package org.fao.geonet.kernel.search.submission.batch; + +import co.elastic.clients.elasticsearch.core.BulkRequest; +import co.elastic.clients.elasticsearch.core.DeleteByQueryRequest; +import org.fao.geonet.index.es.EsRestClient; +import org.fao.geonet.kernel.search.EsSearchManager; +import org.fao.geonet.kernel.search.submission.IDeletionSubmitter; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +public class BatchingDeletionSubmitter extends BatchingSubmitterBase implements IDeletionSubmitter { + + protected static class State extends StateBase { + private final List listOfUUIDsToDelete = new ArrayList<>(); + + @Override + protected void cleanUp() { + // Send any remaining pending documents + if (!listOfUUIDsToDelete.isEmpty()) { + deleteDocumentsFromIndex(listOfUUIDsToDelete); + } + } + + private void deleteDocumentsFromIndex(List toDelete) { + EsRestClient restClient = searchManager.getClient(); + BulkRequest bulkRequest = restClient.buildDeleteBulkRequest(searchManager.getDefaultIndex(), toDelete); + CompletableFuture currentIndexFuture = restClient.getAsyncClient().bulk(bulkRequest) + .thenAcceptAsync(bulkItemResponses -> searchManager.handleDeletionResponse(bulkItemResponses, toDelete)); + queueFuture(currentIndexFuture); + } + + private void deleteDocumentByQuery(String query) { + EsRestClient restClient = searchManager.getClient(); + + DeleteByQueryRequest deleteByQueryRequest = restClient.buildDeleteByQuery(searchManager.getDefaultIndex(), query); + final CompletableFuture deleteByQueryFuture = restClient.getAsyncClient().deleteByQuery(deleteByQueryRequest) + .thenAcceptAsync(response -> searchManager.handleDeletionResponse(response, query)); + queueFuture(deleteByQueryFuture); + } + } + + public BatchingDeletionSubmitter() { + super(new State()); + } + + public BatchingDeletionSubmitter(int estimatedTotalSize) { + super(new State(), estimatedTotalSize); + } + + @Override + public void submitUUIDToIndex(String uuid, EsSearchManager searchManager) { + if (state.closed) { + throw new IllegalStateException("Attempting to use a closed " + this.getClass().getSimpleName()); + } + + state.searchManager = searchManager; + List listOfUUIDsToDelete = state.listOfUUIDsToDelete; + listOfUUIDsToDelete.add(uuid); + if (listOfUUIDsToDelete.size() >= commitInterval) { + List toDelete = new ArrayList<>(listOfUUIDsToDelete); + listOfUUIDsToDelete.clear(); + state.deleteDocumentsFromIndex(toDelete); + } + } + + @Override + public void submitQueryToIndex(String query, EsSearchManager searchManager) { + if (state.closed) { + throw new IllegalStateException("Attempting to use a closed " + this.getClass().getSimpleName()); + } + + state.searchManager = searchManager; + state.deleteDocumentByQuery(query); + } +} diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingIndexSubmitter.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingIndexSubmitter.java new file mode 100644 index 00000000000..80a9f18f0c4 --- /dev/null +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingIndexSubmitter.java @@ -0,0 +1,77 @@ +package org.fao.geonet.kernel.search.submission.batch; + +import co.elastic.clients.elasticsearch.core.BulkRequest; +import org.fao.geonet.index.es.EsRestClient; +import org.fao.geonet.kernel.search.EsSearchManager; +import org.fao.geonet.kernel.search.submission.IIndexSubmitter; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +/** + * An index submitter that batches documents into larger chunks and sends them asynchronously to the index. + * When closing, this submitter sends the remaining item to the index and waits until all elasticsearch requests have been received, + * so after closing there are no pending changes + */ +public class BatchingIndexSubmitter extends BatchingSubmitterBase implements IIndexSubmitter { + protected static class State extends StateBase { + private final Map listOfDocumentsToIndex = new HashMap<>(); + + @Override + protected void cleanUp() { + // Send any remaining pending documents + if (!listOfDocumentsToIndex.isEmpty()) { + sendDocumentsToIndex(listOfDocumentsToIndex); + } + } + + private void sendDocumentsToIndex(Map toIndex) { + EsRestClient restClient = searchManager.getClient(); + BulkRequest bulkRequest = restClient.buildIndexBulkRequest(searchManager.getDefaultIndex(), toIndex); + CompletableFuture currentIndexFuture = restClient.getAsyncClient().bulk(bulkRequest) + .thenAcceptAsync(bulkItemResponses -> { + try { + searchManager.handleIndexResponse(bulkItemResponses, toIndex); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }).exceptionally(e -> { + LOGGER.error( + "An error occurred while indexing {} documents in current indexing list.", + toIndex.size(), e); + return null; + }); + queueFuture(currentIndexFuture); + } + } + + public BatchingIndexSubmitter() { + super(new State()); + } + + /** + * @param estimatedTotalSize The estimated size of documents to index. Does not need to match the actual amount of submitted documents + */ + public BatchingIndexSubmitter(int estimatedTotalSize) { + super(new State(), estimatedTotalSize); + } + + @Override + public void submitToIndex(String id, String jsonDocument, EsSearchManager searchManager) { + if (state.closed) { + throw new IllegalStateException("Attempting to use a closed " + this.getClass().getSimpleName()); + } + + state.searchManager = searchManager; + Map listOfDocumentsToIndex = state.listOfDocumentsToIndex; + listOfDocumentsToIndex.put(id, jsonDocument); + if (listOfDocumentsToIndex.size() >= commitInterval) { + Map toIndex = new HashMap<>(listOfDocumentsToIndex); + listOfDocumentsToIndex.clear(); + state.sendDocumentsToIndex(toIndex); + } + } +} diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingSubmitterBase.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingSubmitterBase.java new file mode 100644 index 00000000000..596896de2a4 --- /dev/null +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingSubmitterBase.java @@ -0,0 +1,51 @@ +package org.fao.geonet.kernel.search.submission.batch; + +import java.lang.ref.Cleaner; +import java.util.concurrent.CompletableFuture; + +public abstract class BatchingSubmitterBase implements AutoCloseable { + private static final Cleaner CLEANER = Cleaner.create(); + protected final STATE state; + protected final int commitInterval; + private final Cleaner.Cleanable cleanable; + + protected BatchingSubmitterBase(final STATE state) { + this.commitInterval = 200; + this.state = state; + this.cleanable = CLEANER.register(this, state); + } + + protected BatchingSubmitterBase(final STATE state, int estimatedTotalSize) { + if (estimatedTotalSize < 0) { + throw new IllegalArgumentException("estimatedTotalSize must not be negative"); + } + + // Compute an ideal commit interval based on estimated size of elements to index + // Try to strike a balance between + // a) Not making enough bulk requests, thus having to wait a long time at the end for a large chunk => try to make at least 8 requests + int elementsPerBatchRequest = estimatedTotalSize / 8; + // b) Making too many requests, adding unnecessary overhead => set the minimum batch size to 20 + elementsPerBatchRequest = Math.max(20, elementsPerBatchRequest); + // c) Growing the listOfDocumentsToIndex too large => set the maximum batch size to 200 + elementsPerBatchRequest = Math.min(200, elementsPerBatchRequest); + this.commitInterval = elementsPerBatchRequest; + this.state = state; + this.cleanable = CLEANER.register(this, state); + } + + @Override + public void close() { + if (this.state.closed) { + throw new IllegalStateException("Attempting to close a already closed " + this.getClass().getSimpleName()); + } + this.state.closed = true; + this.cleanable.clean(); + + // Wait for all remaining documents to be received + for (CompletableFuture inflightFuture : state.inflightFutures) { + if (inflightFuture != null) { + inflightFuture.join(); + } + } + } +} diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/StateBase.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/StateBase.java new file mode 100644 index 00000000000..f816fb4a619 --- /dev/null +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/StateBase.java @@ -0,0 +1,46 @@ +package org.fao.geonet.kernel.search.submission.batch; + +import org.fao.geonet.constants.Geonet; +import org.fao.geonet.kernel.search.EsSearchManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.CompletableFuture; + +public abstract class StateBase implements Runnable { + protected static final Logger LOGGER = LoggerFactory.getLogger(Geonet.INDEX_ENGINE); + /** + * Maximum number of inflight bulk requests before waiting for the Elasticsearch + */ + private static final int MAX_INFLIGHT_INDEX_REQUESTS = 4; + @SuppressWarnings("unchecked") + protected final CompletableFuture[] inflightFutures = new CompletableFuture[MAX_INFLIGHT_INDEX_REQUESTS]; + protected int index = 0; + protected boolean closed = false; + protected EsSearchManager searchManager; + + @Override + public final void run() { + if (!closed) { + LOGGER.error("BatchingIndexSubmittor was not closed before it was cleaned! Sending any remaining data"); + } + cleanUp(); + } + + protected void queueFuture(CompletableFuture newFuture) { + // Request send, sort into queue + // First, see if the previous future already finished + CompletableFuture previousFuture = inflightFutures[index]; + if (previousFuture != null) { + if (!previousFuture.isDone()) { + // Normally, the ES should be able to keep up. If it does not, just wait until there is some space in the ring buffer + LOGGER.info("Waiting for elasticsearch to process pending bulk requests..."); + previousFuture.join(); + } + } + inflightFutures[index] = newFuture; + index = (index + 1) % inflightFutures.length; + } + + protected abstract void cleanUp(); +} diff --git a/core/src/test/java/org/fao/geonet/AbstractCoreIntegrationTest.java b/core/src/test/java/org/fao/geonet/AbstractCoreIntegrationTest.java index 5bd12fcb26f..3a4178df315 100644 --- a/core/src/test/java/org/fao/geonet/AbstractCoreIntegrationTest.java +++ b/core/src/test/java/org/fao/geonet/AbstractCoreIntegrationTest.java @@ -49,6 +49,7 @@ import org.fao.geonet.kernel.mef.Importer; import org.fao.geonet.kernel.mef.MEFLib; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.repository.AbstractSpringDataTest; import org.fao.geonet.repository.GroupRepository; import org.fao.geonet.repository.SourceRepository; @@ -350,7 +351,7 @@ public int importMetadataXML(ServiceContext context, String uuid, InputStream xm id, createDate, createDate, "" + groupId, metadataType); - dataManager.indexMetadata(id.get(0), true); + dataManager.indexMetadata(id.get(0), DirectIndexSubmitter.INSTANCE); return Integer.parseInt(id.get(0)); } @@ -386,11 +387,7 @@ public boolean resetLuceneIndex() { return true; } - protected AbstractMetadata injectMetadataInDbDoNotRefreshHeader(Element sampleMetadataXml, ServiceContext context) throws Exception { - return injectMetadataInDb(sampleMetadataXml, context, false); - } - - protected AbstractMetadata injectMetadataInDb(Element sampleMetadataXml, ServiceContext context, boolean resfreshHeader) throws Exception { + protected AbstractMetadata injectMetadataInDb(Element sampleMetadataXml, ServiceContext context) throws Exception { String uuid = UUID.randomUUID().toString(); String schema = schemaManager.autodetectSchema(sampleMetadataXml); Xml.selectElement(sampleMetadataXml, @@ -419,6 +416,6 @@ protected AbstractMetadata injectMetadataInDb(Element sampleMetadataXml, Service .setHarvested(false); return metadataManager.insertMetadata(context, metadata, sampleMetadataXml, IndexingMode.none, false, UpdateDatestamp.NO, - false, resfreshHeader); + false, DirectIndexSubmitter.INSTANCE); } } diff --git a/core/src/test/java/org/fao/geonet/kernel/AbstractIntegrationTestWithMockedSingletons.java b/core/src/test/java/org/fao/geonet/kernel/AbstractIntegrationTestWithMockedSingletons.java index 2238444e734..10be099be52 100644 --- a/core/src/test/java/org/fao/geonet/kernel/AbstractIntegrationTestWithMockedSingletons.java +++ b/core/src/test/java/org/fao/geonet/kernel/AbstractIntegrationTestWithMockedSingletons.java @@ -7,6 +7,7 @@ import org.fao.geonet.domain.MetadataType; import org.fao.geonet.kernel.datamanager.IMetadataManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.repository.SourceRepository; import org.jdom.Element; import org.springframework.beans.factory.annotation.Autowired; @@ -65,6 +66,6 @@ protected AbstractMetadata insertTemplateResourceInDb(ServiceContext serviceCont false, NO, false, - true); + DirectIndexSubmitter.INSTANCE); } } diff --git a/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java b/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java index f81456e4f84..ee046f77c92 100644 --- a/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java +++ b/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java @@ -41,6 +41,7 @@ import org.fao.geonet.domain.User; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmitter; import org.fao.geonet.repository.GroupRepository; import org.fao.geonet.repository.SourceRepository; import org.fao.geonet.repository.specification.MetadataSpecs; @@ -74,11 +75,11 @@ public void testDeleteMetadata() throws Exception { ServiceContext serviceContext = createContextAndLogAsAdmin(); long count = metadataRepository.count(); - int mdId = injectMetadataInDbDoNotRefreshHeader(getSampleMetadataXml(), serviceContext).getId(); + int mdId = injectMetadataInDb(getSampleMetadataXml(), serviceContext).getId(); assertEquals(count + 1, metadataRepository.count()); - metadataManager.deleteMetadata(serviceContext, String.valueOf(mdId)); + metadataManager.deleteMetadata(serviceContext, String.valueOf(mdId), DirectDeletionSubmitter.INSTANCE); assertEquals(count, metadataRepository.count()); } diff --git a/core/src/test/java/org/fao/geonet/kernel/LocalXLinksUpdateDeleteTest.java b/core/src/test/java/org/fao/geonet/kernel/LocalXLinksUpdateDeleteTest.java index e67834f85ef..66dabc7612a 100644 --- a/core/src/test/java/org/fao/geonet/kernel/LocalXLinksUpdateDeleteTest.java +++ b/core/src/test/java/org/fao/geonet/kernel/LocalXLinksUpdateDeleteTest.java @@ -5,6 +5,7 @@ import org.fao.geonet.domain.AbstractMetadata; import org.fao.geonet.kernel.datamanager.IMetadataManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.utils.Xml; @@ -74,8 +75,8 @@ public void deleteAllowedWhenRefNotExists() throws Exception { AbstractMetadata contactMetadata = insertContact(); AbstractMetadata vicinityMapMetadata = insertVicinityMap(contactMetadata); - metadataManager.deleteMetadata(serviceContext, Integer.toString(vicinityMapMetadata.getId())); - metadataManager.deleteMetadata(serviceContext, Integer.toString(contactMetadata.getId())); + metadataManager.deleteMetadata(serviceContext, Integer.toString(vicinityMapMetadata.getId()), DirectDeletionSubmitter.INSTANCE); + metadataManager.deleteMetadata(serviceContext, Integer.toString(contactMetadata.getId()), DirectDeletionSubmitter.INSTANCE); assertNull(metadataManager.getMetadata(Integer.toString(contactMetadata.getId()))); } @@ -87,7 +88,7 @@ public void deleteHasToBeForbiddenWhenRefExistsAndSettingsSaySo() throws Excepti try { metadataManager.deleteMetadata(serviceContext, - Integer.toString(contactMetadata.getId())); + Integer.toString(contactMetadata.getId()), DirectDeletionSubmitter.INSTANCE); } catch (Exception e) { } @@ -100,7 +101,7 @@ public void deleteHasToBeAllowedWhenRefExistsAndSettingsSaySo() throws Exception AbstractMetadata contactMetadata = insertContact(); insertVicinityMap(contactMetadata); - metadataManager.deleteMetadata(serviceContext, Integer.toString(contactMetadata.getId())); + metadataManager.deleteMetadata(serviceContext, Integer.toString(contactMetadata.getId()), DirectDeletionSubmitter.INSTANCE); assertNull(metadataManager.getMetadata(Integer.toString(contactMetadata.getId()))); } diff --git a/core/src/test/java/org/fao/geonet/kernel/url/UrlAnalyzerTest.java b/core/src/test/java/org/fao/geonet/kernel/url/UrlAnalyzerTest.java index 607ea1f58c6..0a5a3c8c1e1 100644 --- a/core/src/test/java/org/fao/geonet/kernel/url/UrlAnalyzerTest.java +++ b/core/src/test/java/org/fao/geonet/kernel/url/UrlAnalyzerTest.java @@ -11,6 +11,8 @@ import org.fao.geonet.kernel.SchemaManager; import org.fao.geonet.kernel.datamanager.IMetadataManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.repository.LinkRepository; import org.fao.geonet.repository.LinkStatusRepository; @@ -187,7 +189,7 @@ public void deleteMetadaLinkWhenMdDisappeared() throws Exception { SimpleJpaRepository metadataLinkRepository = new SimpleJpaRepository(MetadataLink.class, entityManager); List metadataLinkList = metadataLinkRepository.findAll(); assertEquals(6, metadataLinkList.size()); - dataManager.deleteMetadata(context, md.getId() + ""); + dataManager.deleteMetadata(context, md.getId() + "", DirectDeletionSubmitter.INSTANCE); linkRepository.findAll().stream().forEach(toTest::purgeMetataLink); @@ -301,7 +303,7 @@ private AbstractMetadata insertMetadataInDb(Element element) throws Exception { false, NO, false, - false); + DirectIndexSubmitter.INSTANCE); return dbInsertedMetadata; } diff --git a/csw-server/src/main/java/org/fao/geonet/component/csw/GetRecords.java b/csw-server/src/main/java/org/fao/geonet/component/csw/GetRecords.java index 5b0fb4e224d..abf88a6e550 100644 --- a/csw-server/src/main/java/org/fao/geonet/component/csw/GetRecords.java +++ b/csw-server/src/main/java/org/fao/geonet/component/csw/GetRecords.java @@ -44,6 +44,7 @@ import org.fao.geonet.kernel.csw.services.getrecords.FieldMapper; import org.fao.geonet.kernel.csw.services.getrecords.SearchController; import org.fao.geonet.kernel.csw.services.getrecords.SortByParser; +import org.fao.geonet.kernel.search.index.BatchOpsMetadataReindexer; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.CustomElementSetRepository; @@ -129,10 +130,9 @@ public Element execute(Element request, ServiceContext context) throws CatalogEx // Return exception is indexing. GeonetContext gc = (GeonetContext) context.getHandlerContext(Geonet.CONTEXT_NAME); - DataManager dataManager = gc.getBean(DataManager.class); SettingManager settingsManager = gc.getBean(SettingManager.class); if (!settingsManager.getValueAsBool(SYSTEM_CSW_ENABLEWHENINDEXING) && - dataManager.isIndexing()) { + BatchOpsMetadataReindexer.isIndexing()) { throw new RuntimeException("Catalog is indexing records, retry later."); } diff --git a/csw-server/src/main/java/org/fao/geonet/component/csw/Transaction.java b/csw-server/src/main/java/org/fao/geonet/component/csw/Transaction.java index f571bb968bc..a395ffe1d4c 100644 --- a/csw-server/src/main/java/org/fao/geonet/component/csw/Transaction.java +++ b/csw-server/src/main/java/org/fao/geonet/component/csw/Transaction.java @@ -50,6 +50,8 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.schema.MetadataSchema; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.utils.Log; @@ -279,7 +281,7 @@ private boolean insertTransaction(Element xml, List documents, dataMan.setOperation(context, id, "" + ReservedGroup.all.getId(), ReservedOperation.dynamic); } - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, DirectIndexSubmitter.INSTANCE); AbstractMetadata metadata = metadataUtils.findOne(id); ApplicationContext applicationContext = ApplicationContextHolder.get(); @@ -525,38 +527,40 @@ private int deleteTransaction(Element request, ServiceContext context) throws Ex return deleted; // delete all matching records - while (i.hasNext()) { - Element result = i.next(); - String uuid = result.getChildText("identifier", Csw.NAMESPACE_DC); - String id = dataMan.getMetadataId(uuid); + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter(results.size())) { + while (i.hasNext()) { + Element result = i.next(); + String uuid = result.getChildText("identifier", Csw.NAMESPACE_DC); + String id = dataMan.getMetadataId(uuid); - if (id == null) { - return deleted; - } + if (id == null) { + return deleted; + } - if (!dataMan.getAccessManager().canEdit(context, id)) { - throw new NoApplicableCodeEx("User not allowed to delete metadata : " + id); - } - AbstractMetadata metadata = metadataUtils.findOneByUuid(uuid); - LinkedHashMap titles = new LinkedHashMap<>(); - try { - titles = metadataUtils.extractTitles(Integer.toString(metadata.getId())); - } catch (Exception e) { - Log.warning(Geonet.DATA_MANAGER, - String.format( - "Error while extracting title for the metadata %d " + - "while creating delete event. Error is %s.", - metadata.getId(), e.getMessage())); - } - RecordDeletedEvent recordDeletedEvent = new RecordDeletedEvent( - metadata.getId(), metadata.getUuid(), titles, - context.getUserSession().getUserIdAsInt(), - metadata.getData()); + if (!dataMan.getAccessManager().canEdit(context, id)) { + throw new NoApplicableCodeEx("User not allowed to delete metadata : " + id); + } + AbstractMetadata metadata = metadataUtils.findOneByUuid(uuid); + LinkedHashMap titles = new LinkedHashMap<>(); + try { + titles = metadataUtils.extractTitles(Integer.toString(metadata.getId())); + } catch (Exception e) { + Log.warning(Geonet.DATA_MANAGER, + String.format( + "Error while extracting title for the metadata %d " + + "while creating delete event. Error is %s.", + metadata.getId(), e.getMessage())); + } + RecordDeletedEvent recordDeletedEvent = new RecordDeletedEvent( + metadata.getId(), metadata.getUuid(), titles, + context.getUserSession().getUserIdAsInt(), + metadata.getData()); - metadataManager.deleteMetadata(context, id); - recordDeletedEvent.publish(ApplicationContextHolder.get()); + metadataManager.deleteMetadata(context, id, submitter); + recordDeletedEvent.publish(ApplicationContextHolder.get()); - deleted++; + deleted++; + } } return deleted; diff --git a/csw-server/src/test/java/org/fao/geonet/component/csw/CswTransactionIntegrationTest.java b/csw-server/src/test/java/org/fao/geonet/component/csw/CswTransactionIntegrationTest.java index e53ddb3cd87..be50321e7f3 100644 --- a/csw-server/src/test/java/org/fao/geonet/component/csw/CswTransactionIntegrationTest.java +++ b/csw-server/src/test/java/org/fao/geonet/component/csw/CswTransactionIntegrationTest.java @@ -36,6 +36,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.repository.MetadataRepositoryTest; import org.fao.geonet.utils.Xml; import org.jdom.Content; @@ -48,9 +49,6 @@ import java.io.IOException; import java.nio.file.Path; import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import static org.fao.geonet.constants.Geonet.Namespaces.GCO; @@ -390,7 +388,7 @@ private void addPhotographicMetadataToRepository(int ownerId) throws Exception { extras.put("_isTemplate", "n"); extras.put("_owner", "" + ownerId); _searchManager.index(schemaDir, metadata.getXmlData(false), "" + metadata.getId(), extras, - MetadataType.METADATA,false, IndexingMode.full); + MetadataType.METADATA, DirectIndexSubmitter.INSTANCE, IndexingMode.full); } private Element createUpdateTransaction(String property, Object newValue) { diff --git a/estest/src/test/java/org/fao/geonet/index/es/IndexationTest.java b/estest/src/test/java/org/fao/geonet/index/es/IndexationTest.java deleted file mode 100644 index 38b41744168..00000000000 --- a/estest/src/test/java/org/fao/geonet/index/es/IndexationTest.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.fao.geonet.index.es; - -import org.fao.geonet.domain.MetadataType; -import org.fao.geonet.kernel.search.EsSearchManager; -import org.jdom.Element; -import org.junit.Ignore; -import org.junit.Test; - -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.Map; - -public class IndexationTest { - - @Ignore - @Test - public void nominal() throws Exception { - - Path schemaDir = Paths.get(this.getClass().getResource("/nominal").toURI()); - Element metadata = new Element("root"); - String id = "test-id"; - Map moreFields = new HashMap<>(); - MetadataType metadataType = MetadataType.METADATA; - String root = "dummy"; - boolean forceRefreshReaders = true; - - EsSearchManager toTest = new EsSearchManager(); - -// toTest.index(schemaDir, metadata, id, moreFields, metadataType, root, forceRefreshReaders); -// -// String docToBeIndexed = toTest.listOfDocumentsToIndex.get(id); -// Map result = new ObjectMapper().readValue(docToBeIndexed, HashMap.class); -// Assert.assertEquals(3, result.size()); -// Assert.assertEquals("test-id", result.get("id")); -// Assert.assertEquals("metadata", result.get("docType")); -// Assert.assertEquals("source-from-index-xsl", result.get("sourceCatalogue")); - } -} diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/BaseAligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/BaseAligner.java index 101f6fd78ab..f7e3763f81a 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/BaseAligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/BaseAligner.java @@ -35,6 +35,7 @@ import org.fao.geonet.kernel.harvest.harvester.CategoryMapper; import org.fao.geonet.kernel.harvest.harvester.GroupMapper; import org.fao.geonet.kernel.harvest.harvester.Privileges; +import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.MetadataCategoryRepository; @@ -63,14 +64,16 @@ * * @author heikki doeleman */ -public abstract class BaseAligner

extends AbstractAligner

{ +public abstract class BaseAligner

extends AbstractAligner

implements AutoCloseable { - private static Logger LOGGER = LoggerFactory.getLogger(Geonet.HARVESTER); + private static final Logger LOGGER = LoggerFactory.getLogger(Geonet.HARVESTER); public final AtomicBoolean cancelMonitor; + protected final BatchingIndexSubmitter batchingIndexSubmitter; protected BaseAligner(AtomicBoolean cancelMonitor) { this.cancelMonitor = cancelMonitor; + this.batchingIndexSubmitter = new BatchingIndexSubmitter(); } public void addCategories(AbstractMetadata metadata, Iterable categories, @@ -199,4 +202,8 @@ public Element translateMetadataContent(ServiceContext context, return md; } + @Override + public void close() throws Exception { + this.batchingIndexSubmitter.close(); + } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractHarvester.java index 2398aa96c10..e56b9342930 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractHarvester.java @@ -49,6 +49,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.harvest.Common.OperResult; import org.fao.geonet.kernel.harvest.Common.Status; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.kernel.setting.HarvesterSettingsManager; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; @@ -278,9 +279,12 @@ public void destroy() throws Exception { final Specification ownedByHarvester = Specification.where(MetadataSpecs.hasHarvesterUuid(getParams().getUuid())); Set sources = new HashSet<>(); - for (Integer metadataId : metadataRepository.findAllIdsBy(ownedByHarvester)) { - sources.add(metadataUtils.findOne(metadataId).getSourceInfo().getSourceId()); - metadataManager.deleteMetadata(context, "" + metadataId); + List allIdsBy = metadataRepository.findAllIdsBy(ownedByHarvester); + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter(allIdsBy.size())) { + for (Integer metadataId : allIdsBy) { + sources.add(metadataUtils.findOne(metadataId).getSourceInfo().getSourceId()); + metadataManager.deleteMetadata(context, "" + metadataId, submitter); + } } // Remove all sources related to the harvestUuid if they are not linked to any record anymore diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEHarvester.java index 8b6e7881742..8d8b31265ea 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEHarvester.java @@ -28,7 +28,6 @@ import org.fao.geonet.ApplicationContextHolder; import org.fao.geonet.Logger; import org.fao.geonet.arcgis.ArcSDEConnection; -import org.fao.geonet.constants.Geonet; import org.fao.geonet.domain.AbstractMetadata; import org.fao.geonet.domain.ISODate; import org.fao.geonet.domain.Metadata; @@ -42,6 +41,8 @@ import org.fao.geonet.kernel.harvest.harvester.GroupMapper; import org.fao.geonet.kernel.harvest.harvester.HarvestResult; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.repository.MetadataRepository; import org.fao.geonet.repository.OperationAllowedRepository; import org.fao.geonet.repository.specification.MetadataSpecs; @@ -270,26 +271,25 @@ private void align(Map metadataList) throws Exception { continue; } - BaseAligner aligner = new BaseAligner(cancelMonitor) { - }; - aligner.setParams(params); - // - // add / update the metadata from this harvesting result - // - String id = dataMan.getMetadataId(uuid); - if (id == null) { - id = addMetadata(metadataElement, uuid, schema, localGroups, localCateg, aligner); - result.addedMetadata++; - } else { - updateMetadata(metadataElement, id, localGroups, localCateg, aligner); - result.updatedMetadata++; - } + try (BaseAligner aligner = new BaseAligner<>(cancelMonitor) {}) { + aligner.setParams(params); + // + // add / update the metadata from this harvesting result + // + String id = dataMan.getMetadataId(uuid); + if (id == null) { + id = addMetadata(metadataElement, uuid, schema, localGroups, localCateg, aligner); + result.addedMetadata++; + } else { + updateMetadata(metadataElement, id, localGroups, localCateg, aligner); + result.updatedMetadata++; + } + if (StringUtils.isNotEmpty(thumbnailContent)) { + loadMetadataThumbnail(thumbnailContent, id, uuid); + } - if (StringUtils.isNotEmpty(thumbnailContent)) { - loadMetadataThumbnail(thumbnailContent, id, uuid); + idsForHarvestingResult.add(Integer.valueOf(id)); } - - idsForHarvestingResult.add(Integer.valueOf(id)); } } }catch(Throwable t) { @@ -307,20 +307,22 @@ private void align(Map metadataList) throws Exception { // Set idsResultHs = Sets.newHashSet(idsForHarvestingResult); List existingMetadata = context.getBean(MetadataRepository.class).findIdsBy((Specification) MetadataSpecs.hasHarvesterUuid(params.getUuid())); - for (Integer existingId : existingMetadata) { + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter()) { + for (Integer existingId : existingMetadata) { - if (cancelMonitor.get()) { - return; - } - if (!idsResultHs.contains(existingId)) { - log.debug(" Removing: " + existingId); - metadataManager.deleteMetadata(context, existingId.toString()); - result.locallyRemoved++; + if (cancelMonitor.get()) { + return; + } + if (!idsResultHs.contains(existingId)) { + log.debug(" Removing: " + existingId); + metadataManager.deleteMetadata(context, existingId.toString(), submitter); + result.locallyRemoved++; + } } } } - private void updateMetadata(Element xml, String id, GroupMapper localGroups, final CategoryMapper localCateg, BaseAligner aligner) throws Exception { + private void updateMetadata(Element xml, String id, GroupMapper localGroups, final CategoryMapper localCateg, BaseAligner aligner) throws Exception { log.info("Updating metadata with id: " + id); // @@ -352,14 +354,14 @@ private void updateMetadata(Element xml, String id, GroupMapper localGroups, fin metadataManager.flush(); - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, DirectIndexSubmitter.INSTANCE); } /** * Inserts a metadata into the database. Lucene index is updated after insertion. */ private String addMetadata(Element xml, String uuid, String schema, GroupMapper localGroups, final CategoryMapper localCateg, - BaseAligner aligner) throws Exception { + BaseAligner aligner) throws Exception { log.info(" - Adding metadata with remote uuid: " + uuid); // @@ -396,13 +398,13 @@ private String addMetadata(Element xml, String uuid, String schema, GroupMapper aligner.addCategories(metadata, params.getCategories(), localCateg, context, null, false); - metadata = metadataManager.insertMetadata(context, metadata, xml, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, xml, IndexingMode.none, false, UpdateDatestamp.NO, false, DirectIndexSubmitter.INSTANCE); String id = String.valueOf(metadata.getId()); aligner.addPrivileges(id, params.getPrivileges(), localGroups, context); - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, DirectIndexSubmitter.INSTANCE); return id; } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java index 5097d9a600c..e0a9359e378 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java @@ -57,6 +57,7 @@ import org.fao.geonet.kernel.schema.MetadataSchema; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.utils.Xml; import org.jdom.Element; @@ -256,15 +257,16 @@ public HarvestResult cleanupRemovedRecords(Set records) throws Exception return result; } - for (String uuid : localUuids.getUUIDs()) { - if (!records.contains(uuid)) { - String id = localUuids.getID(uuid); - log.debug(" - Removing old metadata with local id:" + id); - metadataManager.deleteMetadata(context, id); - result.locallyRemoved++; + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter()) { + for (String uuid : localUuids.getUUIDs()) { + if (!records.contains(uuid)) { + String id = localUuids.getID(uuid); + log.debug(" - Removing old metadata with local id:" + id); + metadataManager.deleteMetadata(context, id, submitter); + result.locallyRemoved++; + } } } - dataMan.forceIndexChanges(); return result; } @@ -365,13 +367,13 @@ private void addMetadata(RecordInfo ri, String uuidToAssign) throws Exception { addCategories(metadata, params.getCategories(), localCateg, context, null, false); - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmitter); String id = String.valueOf(metadata.getId()); addPrivileges(id, params.getPrivileges(), localGroups, context); - metadataIndexer.indexMetadata(id, true, IndexingMode.full); + metadataIndexer.indexMetadata(id, batchingIndexSubmitter, IndexingMode.full); result.addedMetadata++; } @@ -435,7 +437,7 @@ private void updateMetadata(RecordInfo ri, String id, boolean force) throws Exce } else { log.debug(" - Updating local metadata for uuid:" + ri.uuid); if (updatingLocalMetadata(ri, id, force)) { - metadataIndexer.indexMetadata(id, true, IndexingMode.full); + metadataIndexer.indexMetadata(id, batchingIndexSubmitter, IndexingMode.full); result.updatedMetadata++; } } @@ -541,7 +543,7 @@ private Element retrieveMetadata(String uuid) { params.getValidate().validate(dataMan, context, response, groupIdVal); } catch (Exception e) { - log.info("Ignoring invalid metadata with uuid " + uuid); + log.info("Ignoring invalid metadata with uuid " + uuid, e); result.doesNotValidate++; return null; } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Harvester.java index fa088a5a4cb..e42b5e8dcb1 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Harvester.java @@ -110,9 +110,8 @@ public HarvestResult harvest(Logger log) throws Exception { boolean error = false; HarvestResult result = new HarvestResult(); - Set uuids = new HashSet(); - try { - Aligner aligner = new Aligner(cancelMonitor, context, server, params, log); + Set uuids = new HashSet<>(); + try (Aligner aligner = new Aligner(cancelMonitor, context, server, params, log)) { searchAndAlign(server, uuids, aligner, errors); result = aligner.cleanupRemovedRecords(uuids); } catch (Exception t) { diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/fragment/FragmentHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/fragment/FragmentHarvester.java index cea522c6567..7623fa3233e 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/fragment/FragmentHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/fragment/FragmentHarvester.java @@ -403,14 +403,14 @@ private void createSubtemplate(String schema, Element md, String uuid, String ti addCategories(metadata, params.categories, localCateg, context, null, false); - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmitter); String id = String.valueOf(metadata.getId()); // Note: we use fragmentAllPrivs here because subtemplates need to be // visible/accessible to all addPrivileges(id, fragmentAllPrivs, localGroups, context); - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, batchingIndexSubmitter); metadataManager.flush(); @@ -594,7 +594,7 @@ private void update(String id, Element template, String title, boolean isSubtemp } dataMan.setHarvestedExt(iId, params.uuid, Optional.of(harvestUri)); - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, batchingIndexSubmitter); metadataManager.flush(); } @@ -634,7 +634,7 @@ private void createMetadata(String recUuid, Element template) throws Exception, } metadata.getCategories().add(metadataCategory); } - metadata = metadataManager.insertMetadata(context, metadata, template, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, template, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmitter); String id = String.valueOf(metadata.getId()); @@ -643,7 +643,7 @@ private void createMetadata(String recUuid, Element template) throws Exception, } addPrivileges(id, params.privileges, localGroups, context); - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, batchingIndexSubmitter); if (log.isDebugEnabled()) { log.debug(" - Commit " + id); diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java index 0a3c5b7bd9d..9e0ed90f72c 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java @@ -51,6 +51,7 @@ import org.fao.geonet.kernel.harvest.harvester.RecordInfo; import org.fao.geonet.kernel.harvest.harvester.UUIDMapper; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.repository.OperationAllowedRepository; import org.fao.geonet.utils.GeonetHttpRequestFactory; import org.fao.geonet.utils.Xml; @@ -105,21 +106,23 @@ public HarvestResult align(Set records, List errors) t //----------------------------------------------------------------------- //--- remove old metadata - for (String uuid : localUuids.getUUIDs()) { - if (cancelMonitor.get()) { - return result; - } + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter()) { + for (String uuid : localUuids.getUUIDs()) { + if (cancelMonitor.get()) { + return result; + } - if (!exists(records, uuid)) { - String id = localUuids.getID(uuid); + if (!exists(records, uuid)) { + String id = localUuids.getID(uuid); - if (log.isDebugEnabled()) - log.debug(" - Removing old metadata with local id:" + id); - metadataManager.deleteMetadata(context, id); + if (log.isDebugEnabled()) + log.debug(" - Removing old metadata with local id:" + id); + metadataManager.deleteMetadata(context, id, submitter); - metadataManager.flush(); + metadataManager.flush(); - result.locallyRemoved++; + result.locallyRemoved++; + } } } @@ -145,8 +148,6 @@ public HarvestResult align(Set records, List errors) t } } - dataMan.forceIndexChanges(); - log.info("End of alignment for : " + params.getName()); return result; @@ -191,13 +192,13 @@ private void addMetadata(RecordInfo ri) throws Exception { addCategories(metadata, params.getCategories(), localCateg, context, null, false); - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmitter); String id = String.valueOf(metadata.getId()); addPrivileges(id, params.getPrivileges(), localGroups, context); - dataMan.indexMetadata(id, Math.random() < 0.01); + dataMan.indexMetadata(id, batchingIndexSubmitter); result.addedMetadata++; } @@ -241,7 +242,7 @@ private void updateMetadata(RecordInfo ri, String id) throws Exception { addCategories(metadata, params.getCategories(), localCateg, context, null, true); metadataManager.flush(); - dataMan.indexMetadata(id, Math.random() < 0.01); + dataMan.indexMetadata(id, batchingIndexSubmitter); result.updatedMetadata++; } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Harvester.java index 8d519a1b2d5..3f457979600 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Harvester.java @@ -151,9 +151,9 @@ public HarvestResult harvest(Logger log) throws Exception { //--- align local node - Aligner aligner = new Aligner(cancelMonitor, log, context, params); - - return aligner.align(records, errors); + try (Aligner aligner = new Aligner(cancelMonitor, log, context, params)) { + return aligner.align(records, errors); + } } /** diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java index 4f6418d5ddf..1b31e15ac82 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java @@ -44,6 +44,7 @@ import org.fao.geonet.kernel.harvest.harvester.*; import org.fao.geonet.kernel.mef.*; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.GroupRepository; @@ -150,24 +151,26 @@ public HarvestResult align(SortedSet records, List err //----------------------------------------------------------------------- //--- remove old metadata - for (String uuid : localUuids.getUUIDs()) { - if (cancelMonitor.get()) { - return this.result; - } + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter()) { + for (String uuid : localUuids.getUUIDs()) { + if (cancelMonitor.get()) { + return this.result; + } - try { - if (!exists(records, uuid)) { - String id = localUuids.getID(uuid); + try { + if (!exists(records, uuid)) { + String id = localUuids.getID(uuid); - if (log.isDebugEnabled()) log.debug(" - Removing old metadata with id:" + id); - metadataManager.deleteMetadata(context, id); + if (log.isDebugEnabled()) log.debug(" - Removing old metadata with id:" + id); + metadataManager.deleteMetadata(context, id, submitter); - result.locallyRemoved++; + result.locallyRemoved++; + } + } catch (Exception t) { + log.error("Couldn't remove metadata with uuid " + uuid); + log.error(t); + result.unchangedMetadata++; } - } catch (Exception t) { - log.error("Couldn't remove metadata with uuid " + uuid); - log.error(t); - result.unchangedMetadata++; } } //--- insert/update new metadata @@ -255,8 +258,6 @@ public HarvestResult align(SortedSet records, List err } } - dataMan.forceIndexChanges(); - log.info("End of alignment for : " + params.getName()); return result; @@ -517,7 +518,7 @@ private String addMetadata(RecordInfo ri, Element md, Element info, boolean loca addCategories(metadata, params.getCategories(), localCateg, context, null, false); - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, ufo, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, ufo, UpdateDatestamp.NO, false, this.batchingIndexSubmitter); String id = String.valueOf(metadata.getId()); @@ -546,7 +547,7 @@ private String addMetadata(RecordInfo ri, Element md, Element info, boolean loca } context.getBean(IMetadataManager.class).save(metadata); - dataMan.indexMetadata(id, Math.random() < 0.01); + dataMan.indexMetadata(id, this.batchingIndexSubmitter); result.addedMetadata++; return id; @@ -843,7 +844,7 @@ private void updateMetadata(RecordInfo ri, String id, Element md, metadataManager.save(metadata); // dataMan.flush(); - dataMan.indexMetadata(id, Math.random() < 0.01); + dataMan.indexMetadata(id, this.batchingIndexSubmitter); } private void handleFile(String id, String file, MetadataResourceVisibility visibility, String changeDate, diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Harvester.java index 96ac8c5660b..ec5a74f5157 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Harvester.java @@ -226,8 +226,9 @@ public HarvestResult harvest(Logger log) throws Exception { HarvestResult result = new HarvestResult(); if (!error) { try { - Aligner aligner = new Aligner(cancelMonitor, log, context, req, params, remoteInfo); - result = aligner.align(records, errors); + try (Aligner aligner = new Aligner(cancelMonitor, log, context, req, params, remoteInfo)) { + result = aligner.align(records, errors); + } Map sources = buildSources(remoteInfo); updateSources(records, sources); diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/Aligner.java index 93f48ff9a48..0c64cd2844c 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/Aligner.java @@ -42,6 +42,8 @@ import org.fao.geonet.kernel.harvest.harvester.HarvestResult; import org.fao.geonet.kernel.harvest.harvester.UUIDMapper; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.repository.MetadataCategoryRepository; import org.fao.geonet.repository.specification.MetadataCategorySpecs; import org.fao.geonet.utils.XmlRequest; @@ -104,19 +106,21 @@ public HarvestResult align(Element result, String siteId) throws Exception { //----------------------------------------------------------------------- //--- remove old metadata - for (String uuid : localUuids.getUUIDs()) { - if (cancelMonitor.get()) { - return this.result; - } + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter()) { + for (String uuid : localUuids.getUUIDs()) { + if (cancelMonitor.get()) { + return this.result; + } - if (!exists(mdList, uuid)) { - String id = localUuids.getID(uuid); + if (!exists(mdList, uuid)) { + String id = localUuids.getID(uuid); - if (log.isDebugEnabled()) log.debug(" - Removing old metadata with id=" + id); - metadataManager.deleteMetadata(context, id); + if (log.isDebugEnabled()) log.debug(" - Removing old metadata with id=" + id); + metadataManager.deleteMetadata(context, id, submitter); - metadataManager.flush(); - this.result.locallyRemoved++; + metadataManager.flush(); + this.result.locallyRemoved++; + } } } //----------------------------------------------------------------------- @@ -157,7 +161,7 @@ public HarvestResult align(Element result, String siteId) throws Exception { //--- maybe the metadata was unretrievable if (id != null) { - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, DirectIndexSubmitter.INSTANCE); } } } @@ -212,7 +216,7 @@ private String addMetadata(Element info) throws Exception { List categories = info.getChildren("category"); addCategories(metadata, categories); - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, DirectIndexSubmitter.INSTANCE); String id = String.valueOf(metadata.getId()); diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemHarvester.java index 640ddbee67b..febd1658f67 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemHarvester.java @@ -44,6 +44,8 @@ import org.fao.geonet.kernel.harvest.harvester.GroupMapper; import org.fao.geonet.kernel.harvest.harvester.HarvestResult; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.repository.MetadataRepository; import org.fao.geonet.repository.specification.MetadataSpecs; import org.fao.geonet.utils.IO; @@ -53,8 +55,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import java.util.*; - /** * Harvester for local filesystem. * @@ -90,48 +90,52 @@ protected LocalFilesystemParams createParams() { */ private HarvestResult align(Path root) throws Exception { log.debug("Start of alignment for : " + params.getName()); - final LocalFsHarvesterFileVisitor visitor = new LocalFsHarvesterFileVisitor(cancelMonitor, context, params, this); - if (params.recurse) { - Files.walkFileTree(root, visitor); - } else { - try (DirectoryStream paths = Files.newDirectoryStream(root)) { - for (Path path : paths) { - if (path != null && Files.isRegularFile(path)) { - visitor.visitFile(path, Files.readAttributes(path, BasicFileAttributes.class)); + try (final LocalFsHarvesterFileVisitor visitor = new LocalFsHarvesterFileVisitor(cancelMonitor, context, params, this)) { + if (params.recurse) { + Files.walkFileTree(root, visitor); + } else { + try (DirectoryStream paths = Files.newDirectoryStream(root)) { + for (Path path : paths) { + if (path != null && Files.isRegularFile(path)) { + visitor.visitFile(path, Files.readAttributes(path, BasicFileAttributes.class)); + } } } } - } - result = visitor.getResult(); - log.debug(String.format("Scan directory is done. %d files analyzed.", result.totalMetadata)); - Set idsForHarvestingResult = visitor.getListOfRecords(); - Set idsResultHs = Sets.newHashSet(idsForHarvestingResult); - - if (!params.nodelete) { - log.debug("Starting to delete locally existing metadata " + - "from the same source if they " + - " were not in this harvesting result..."); - List existingMetadata = context.getBean(MetadataRepository.class).findIdsBy((Specification) MetadataSpecs.hasHarvesterUuid(params.getUuid())); - for (Integer existingId : existingMetadata) { - - if (cancelMonitor.get()) { - return this.result; - } - if (!idsResultHs.contains(existingId)) { - log.debug(" Removing: " + existingId); - metadataManager.deleteMetadata(context, existingId.toString()); - result.locallyRemoved++; + result = visitor.getResult(); + log.debug(String.format("Scan directory is done. %d files analyzed.", result.totalMetadata)); + Set idsForHarvestingResult = visitor.getListOfRecords(); + Set idsResultHs = Sets.newHashSet(idsForHarvestingResult); + + if (!params.nodelete) { + log.debug("Starting to delete locally existing metadata " + + "from the same source if they " + + " were not in this harvesting result..."); + List existingMetadata = context.getBean(MetadataRepository.class).findIdsBy((Specification) MetadataSpecs.hasHarvesterUuid(params.getUuid())); + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter()) { + for (Integer existingId : existingMetadata) { + + if (cancelMonitor.get()) { + return this.result; + } + if (!idsResultHs.contains(existingId)) { + log.debug(" Removing: " + existingId); + metadataManager.deleteMetadata(context, existingId.toString(), submitter); + result.locallyRemoved++; + } + } } } - } - log.debug("Starting indexing in batch thread pool..."); + log.debug("Starting indexing in batch thread pool..."); + + List listOfRecordsToIndex = Lists.newArrayList(visitor.getListOfRecordsToIndex()); - List listOfRecordsToIndex = Lists.newArrayList(visitor.getListOfRecordsToIndex()); - log.debug(String.format( - "Starting indexing in batch thread pool of %d updated records ...", - listOfRecordsToIndex.size())); - dataMan.batchIndexInThreadPool(context, listOfRecordsToIndex); + log.debug(String.format( + "Starting indexing in batch thread pool of %d updated records ...", + listOfRecordsToIndex.size())); + dataMan.batchIndexInThreadPool(context, listOfRecordsToIndex); + } log.debug("End of alignment for : " + params.getName()); return result; @@ -170,7 +174,7 @@ void updateMetadata(Element xml, final String id, GroupMapper localGroups, final metadataManager.flush(); - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, DirectIndexSubmitter.INSTANCE); } String addMetadata(Element xml, String uuid, String schema, GroupMapper localGroups, final CategoryMapper localCateg, @@ -218,7 +222,7 @@ String addMetadata(Element xml, String uuid, String schema, GroupMapper localGro aligner.addCategories(metadata, params.getCategories(), localCateg, context, null, false); - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, DirectIndexSubmitter.INSTANCE); String id = String.valueOf(metadata.getId()); @@ -227,7 +231,7 @@ String addMetadata(Element xml, String uuid, String schema, GroupMapper localGro metadataManager.flush(); if (index) { - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, DirectIndexSubmitter.INSTANCE); } return id; } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFsHarvesterFileVisitor.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFsHarvesterFileVisitor.java index 791f9a17cda..172e07eda9f 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFsHarvesterFileVisitor.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFsHarvesterFileVisitor.java @@ -72,7 +72,7 @@ /** * @author Jesse on 11/6/2014. */ -class LocalFsHarvesterFileVisitor extends SimpleFileVisitor { +class LocalFsHarvesterFileVisitor extends SimpleFileVisitor implements AutoCloseable { private final LocalFilesystemParams params; private final DataManager dataMan; @@ -503,4 +503,9 @@ public Set getListOfRecords() { public Set getListOfRecordsToIndex() { return listOfRecordsToIndex; } + + @Override + public void close() throws Exception { + this.aligner.close(); + } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java index 79bc1fb174b..57209b1181f 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java @@ -353,7 +353,6 @@ private void align(XmlRequest t, Set records) throws Exception { } } - dataMan.forceIndexChanges(); log.info("End of alignment for : " + params.getName()); } @@ -431,7 +430,7 @@ private void addMetadata(XmlRequest t, RecordInfo ri, String processName, Map addMetadata(Element capa) throws Exception { if (!dataMan.existsMetadataUuid(uuid)) { result.addedMetadata++; - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmitter); } else { result.updatedMetadata++; String id = dataMan.getMetadataId(uuid); @@ -834,7 +837,7 @@ private WxSLayerRegistry addLayerMetadata(Element layer, Element capa) throws JD } if (!dataMan.existsMetadataUuid(reg.uuid)) { result.addedMetadata++; - metadata = metadataManager.insertMetadata(context, metadata, xml, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, xml, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmitter); } else { result.updatedMetadata++; String id = dataMan.getMetadataId(reg.uuid); diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/ogcwxs/OgcWxSHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/ogcwxs/OgcWxSHarvester.java index 39bcaee8ea1..148249b4a7c 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/ogcwxs/OgcWxSHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/ogcwxs/OgcWxSHarvester.java @@ -74,8 +74,9 @@ protected void storeNodeExtra(OgcWxSParams params, String path, //--------------------------------------------------------------------------- public void doHarvest(Logger log) throws Exception { - Harvester h = new Harvester(cancelMonitor, log, context, params); - result = h.harvest(log); + try (Harvester h = new Harvester(cancelMonitor, log, context, params)) { + result = h.harvest(log); + } } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Aligner.java index 4bb78e3dee8..141fdb7c78d 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Aligner.java @@ -44,6 +44,7 @@ import org.fao.geonet.kernel.harvest.harvester.HarvestResult; import org.fao.geonet.kernel.harvest.harvester.UUIDMapper; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.repository.OperationAllowedRepository; import org.jdom.Element; @@ -178,15 +179,17 @@ public HarvestResult cleanupRemovedRecords(Set records) throws Exception return result; } - for (String uuid : localUuids.getUUIDs()) { - if (!records.contains(uuid)) { - String id = localUuids.getID(uuid); - log.debug(" - Removing old metadata with local id:" + id); - metadataManager.deleteMetadata(context, id); - result.locallyRemoved ++; + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter()) { + + for (String uuid : localUuids.getUUIDs()) { + if (!records.contains(uuid)) { + String id = localUuids.getID(uuid); + log.debug(" - Removing old metadata with local id:" + id); + metadataManager.deleteMetadata(context, id, submitter); + result.locallyRemoved++; + } } } - dataMan.forceIndexChanges(); return result; } @@ -249,13 +252,13 @@ private void addMetadata(Map.Entry recordInfo, String overrideU addCategories(metadata, params.getCategories(), localCateg, context, null, false); - metadata = metadataManager.insertMetadata(context, metadata, xml, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, xml, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmitter); String id = String.valueOf(metadata.getId()); addPrivileges(id, params.getPrivileges(), localGroups, context); - metadataIndexer.indexMetadata(id, true, IndexingMode.full); + metadataIndexer.indexMetadata(id, batchingIndexSubmitter, IndexingMode.full); result.addedMetadata++; } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Harvester.java index 254fac91f84..546b0658bb3 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Harvester.java @@ -104,105 +104,106 @@ public HarvestResult harvest(Logger log) throws Exception { String[] urlList = params.url.split("\n"); boolean error = false; - Aligner aligner = new Aligner(cancelMonitor, context, params, log); - Set listOfUuids = new HashSet<>(); - - for (String url : urlList) { - log.debug("Loading URL: " + url); - String content = retrieveUrl(url); - if (cancelMonitor.get()) { - return new HarvestResult(); - } - log.debug("Response is: " + content); + try (Aligner aligner = new Aligner(cancelMonitor, context, params, log)) { + Set listOfUuids = new HashSet<>(); + + for (String url : urlList) { + log.debug("Loading URL: " + url); + String content = retrieveUrl(url); + if (cancelMonitor.get()) { + return new HarvestResult(); + } + log.debug("Response is: " + content); - int numberOfRecordsToHarvest = -1; + int numberOfRecordsToHarvest = -1; - ObjectMapper objectMapper = new ObjectMapper(); - JsonNode jsonObj = null; - Element xmlObj = null; - SimpleUrlResourceType type; + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode jsonObj = null; + Element xmlObj = null; + SimpleUrlResourceType type; - if (isRDFLike(content)) type = SimpleUrlResourceType.RDFXML; - else if (isXMLLike(content)) type = SimpleUrlResourceType.XML; - else type = SimpleUrlResourceType.JSON; + if (isRDFLike(content)) type = SimpleUrlResourceType.RDFXML; + else if (isXMLLike(content)) type = SimpleUrlResourceType.XML; + else type = SimpleUrlResourceType.JSON; - if (type == SimpleUrlResourceType.XML - || type == SimpleUrlResourceType.RDFXML) { - xmlObj = Xml.loadString(content, false); - } else { - jsonObj = objectMapper.readTree(content); - } - - // TODO: Add page support for Hydra in RDFXML feeds ? - if (StringUtils.isNotEmpty(params.numberOfRecordPath)) { - try { - if (type == SimpleUrlResourceType.XML) { - Object element = Xml.selectSingle(xmlObj, params.numberOfRecordPath, xmlObj.getAdditionalNamespaces()); - if (element != null) { - String s = getXmlElementTextValue(element); - numberOfRecordsToHarvest = Integer.parseInt(s); - } - } else if (type == SimpleUrlResourceType.JSON) { - numberOfRecordsToHarvest = jsonObj.at(params.numberOfRecordPath).asInt(); - } - log.debug("Number of records to harvest: " + numberOfRecordsToHarvest); - } catch (Exception e) { - errors.add(new HarvestError(context, e)); - log.error(String.format("Failed to extract total in response at path %s. Error is: %s", - params.numberOfRecordPath, e.getMessage())); + if (type == SimpleUrlResourceType.XML + || type == SimpleUrlResourceType.RDFXML) { + xmlObj = Xml.loadString(content, false); + } else { + jsonObj = objectMapper.readTree(content); } - } - try { - List listOfUrlForPages = buildListOfUrl(params, numberOfRecordsToHarvest); - for (int i = 0; i < listOfUrlForPages.size(); i++) { - if (i != 0) { - content = retrieveUrl(listOfUrlForPages.get(i)); + + // TODO: Add page support for Hydra in RDFXML feeds ? + if (StringUtils.isNotEmpty(params.numberOfRecordPath)) { + try { if (type == SimpleUrlResourceType.XML) { - xmlObj = Xml.loadString(content, false); - } else { - jsonObj = objectMapper.readTree(content); + Object element = Xml.selectSingle(xmlObj, params.numberOfRecordPath, xmlObj.getAdditionalNamespaces()); + if (element != null) { + String s = getXmlElementTextValue(element); + numberOfRecordsToHarvest = Integer.parseInt(s); + } + } else if (type == SimpleUrlResourceType.JSON) { + numberOfRecordsToHarvest = jsonObj.at(params.numberOfRecordPath).asInt(); } + log.debug("Number of records to harvest: " + numberOfRecordsToHarvest); + } catch (Exception e) { + errors.add(new HarvestError(context, e)); + log.error(String.format("Failed to extract total in response at path %s. Error is: %s", + params.numberOfRecordPath, e.getMessage())); } - if (StringUtils.isNotEmpty(params.loopElement) - || type == SimpleUrlResourceType.RDFXML) { - Map uuids = new HashMap<>(); - try { + } + try { + List listOfUrlForPages = buildListOfUrl(params, numberOfRecordsToHarvest); + for (int i = 0; i < listOfUrlForPages.size(); i++) { + if (i != 0) { + content = retrieveUrl(listOfUrlForPages.get(i)); if (type == SimpleUrlResourceType.XML) { - collectRecordsFromXml(xmlObj, uuids, aligner); - } else if (type == SimpleUrlResourceType.RDFXML) { - collectRecordsFromRdf(xmlObj, uuids, aligner); - } else if (type == SimpleUrlResourceType.JSON) { - collectRecordsFromJson(jsonObj, uuids, aligner); + xmlObj = Xml.loadString(content, false); + } else { + jsonObj = objectMapper.readTree(content); + } + } + if (StringUtils.isNotEmpty(params.loopElement) + || type == SimpleUrlResourceType.RDFXML) { + Map uuids = new HashMap<>(); + try { + if (type == SimpleUrlResourceType.XML) { + collectRecordsFromXml(xmlObj, uuids, aligner); + } else if (type == SimpleUrlResourceType.RDFXML) { + collectRecordsFromRdf(xmlObj, uuids, aligner); + } else if (type == SimpleUrlResourceType.JSON) { + collectRecordsFromJson(jsonObj, uuids, aligner); + } + aligner.align(uuids, errors); + listOfUuids.addAll(uuids.keySet()); + } catch (Exception e) { + errors.add(new HarvestError(this.context, e)); + log.error(String.format("Failed to collect record in response at path %s. Error is: %s", + params.loopElement, e.getMessage())); } - aligner.align(uuids, errors); - listOfUuids.addAll(uuids.keySet()); - } catch (Exception e) { - errors.add(new HarvestError(this.context, e)); - log.error(String.format("Failed to collect record in response at path %s. Error is: %s", - params.loopElement, e.getMessage())); } } + } catch (Exception t) { + error = true; + log.error("Unknown error trying to harvest"); + log.error(t.getMessage()); + log.error(t); + errors.add(new HarvestError(context, t)); + } catch (Throwable t) { + error = true; + log.fatal("Something unknown and terrible happened while harvesting"); + log.fatal(t.getMessage()); + errors.add(new HarvestError(context, t)); } - } catch (Exception t) { - error = true; - log.error("Unknown error trying to harvest"); - log.error(t.getMessage()); - log.error(t); - errors.add(new HarvestError(context, t)); - } catch (Throwable t) { - error = true; - log.fatal("Something unknown and terrible happened while harvesting"); - log.fatal(t.getMessage()); - errors.add(new HarvestError(context, t)); - } - log.info("Total records processed in all searches :" + listOfUuids.size()); - if (error) { - log.warning("Due to previous errors the align process has not been called"); + log.info("Total records processed in all searches :" + listOfUuids.size()); + if (error) { + log.warning("Due to previous errors the align process has not been called"); + } } + aligner.cleanupRemovedRecords(listOfUuids); + return aligner.getResult(); } - aligner.cleanupRemovedRecords(listOfUuids); - return aligner.getResult(); } private void collectRecordsFromJson(JsonNode jsonObj, diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/Harvester.java index 80304603e62..95f41169f5f 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/Harvester.java @@ -56,6 +56,8 @@ import org.fao.geonet.kernel.harvest.harvester.RecordInfo; import org.fao.geonet.kernel.harvest.harvester.UriMapper; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.lib.Lib; import org.fao.geonet.repository.MetadataCategoryRepository; import org.fao.geonet.util.Sha1Encoder; @@ -90,7 +92,6 @@ import java.net.MalformedURLException; import java.net.URL; import java.net.URLEncoder; -import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; @@ -288,27 +289,30 @@ public HarvestResult harvest(Logger log) throws Exception { harvestCatalog(xml); //--- Remove previously harvested metadata for uris that no longer exist on the remote site - for (String localUri : localUris.getUris()) { - if (cancelMonitor.get()) { - return this.result; - } - - if (!harvestUris.contains(localUri)) { - for (RecordInfo record : localUris.getRecords(localUri)) { - if (cancelMonitor.get()) { - return this.result; - } + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter()) { - if (log.isDebugEnabled()) - log.debug(" - Removing deleted metadata with id: " + record.id); - mdManager.deleteMetadata(context, record.id); + for (String localUri : localUris.getUris()) { + if (cancelMonitor.get()) { + return this.result; + } - if (record.isTemplate.equals("s")) { - //--- Uncache xlinks if a subtemplate - Processor.uncacheXLinkUri(metadataGetService + record.uuid); - result.subtemplatesRemoved++; - } else { - result.locallyRemoved++; + if (!harvestUris.contains(localUri)) { + for (RecordInfo record : localUris.getRecords(localUri)) { + if (cancelMonitor.get()) { + return this.result; + } + + if (log.isDebugEnabled()) + log.debug(" - Removing deleted metadata with id: " + record.id); + mdManager.deleteMetadata(context, record.id, submitter); + + if (record.isTemplate.equals("s")) { + //--- Uncache xlinks if a subtemplate + Processor.uncacheXLinkUri(metadataGetService + record.uuid); + result.subtemplatesRemoved++; + } else { + result.locallyRemoved++; + } } } } @@ -572,13 +576,13 @@ private void saveMetadata(Element md, String uuid, String uri, boolean isService addCategories(metadata, params.getCategories(), localCateg, context, null, false); } - metadata = (Metadata) mdManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = (Metadata) mdManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, DirectIndexSubmitter.INSTANCE); String id = String.valueOf(metadata.getId()); addPrivileges(id, params.getPrivileges(), localGroups, context); - mdIndexer.indexMetadata(id, true, IndexingMode.full); + mdIndexer.indexMetadata(id, DirectIndexSubmitter.INSTANCE, IndexingMode.full); mdManager.flush(); } @@ -784,12 +788,14 @@ private void deleteExistingMetadata(String uri) throws Exception { if (localRecords == null) return; - for (RecordInfo record : localRecords) { - mdManager.deleteMetadata(context, record.id); + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter(localRecords.size())) { + for (RecordInfo record : localRecords) { + mdManager.deleteMetadata(context, record.id, submitter); - if (record.isTemplate.equals("s")) { - //--- Uncache xlinks if a subtemplate - Processor.uncacheXLinkUri(metadataGetService + record.uuid); + if (record.isTemplate.equals("s")) { + //--- Uncache xlinks if a subtemplate + Processor.uncacheXLinkUri(metadataGetService + record.uuid); + } } } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/ThreddsHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/ThreddsHarvester.java index 32ca2ff5296..2b5c7e7324c 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/ThreddsHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/ThreddsHarvester.java @@ -72,7 +72,8 @@ protected void storeNodeExtra(ThreddsParams params, String path, //--------------------------------------------------------------------------- public void doHarvest(Logger log) throws Exception { - Harvester h = new Harvester(cancelMonitor, log, context, params); - result = h.harvest(log); + try (Harvester h = new Harvester(cancelMonitor, log, context, params)) { + result = h.harvest(log); + } } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/Harvester.java index cf8717e5213..e0c7040aa43 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/Harvester.java @@ -291,7 +291,7 @@ private void addMetadata(RemoteFile rf) throws Exception { if (StringUtils.isNotEmpty(params.xslfilter)) { md = HarvesterUtil.processMetadata(dataMan.getSchema(schema), md, processName, processParams); - + schema = dataMan.autodetectSchema(md); } @@ -340,14 +340,14 @@ private void addMetadata(RemoteFile rf) throws Exception { } catch (NumberFormatException e) { } - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmitter); String id = String.valueOf(metadata.getId()); addPrivileges(id, params.getPrivileges(), localGroups, context); metadataManager.flush(); - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, batchingIndexSubmitter); result.addedMetadata++; } @@ -522,7 +522,7 @@ private void updateMetadata(RemoteFile rf, RecordInfo recordInfo, boolean force) dataMan.flush(); - dataMan.indexMetadata(recordInfo.id, true); + dataMan.indexMetadata(recordInfo.id, batchingIndexSubmitter); } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WebDavHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WebDavHarvester.java index e745a5b3311..4a87b8573a3 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WebDavHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WebDavHarvester.java @@ -47,8 +47,9 @@ protected void storeNodeExtra(WebDavParams params, String path, String siteId, S public void doHarvest(Logger log) throws Exception { log.info("WebDav doHarvest start"); - Harvester h = new Harvester(cancelMonitor, log, context, params); - result = h.harvest(log); + try (Harvester h = new Harvester(cancelMonitor, log, context, params)) { + result = h.harvest(log); + } log.info("WebDav doHarvest end"); } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/wfsfeatures/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/wfsfeatures/Harvester.java index a0d72818fac..b7ac6c320c9 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/wfsfeatures/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/wfsfeatures/Harvester.java @@ -44,6 +44,7 @@ import org.fao.geonet.kernel.harvest.harvester.fragment.FragmentHarvester; import org.fao.geonet.kernel.harvest.harvester.fragment.FragmentHarvester.FragmentParams; import org.fao.geonet.kernel.harvest.harvester.fragment.FragmentHarvester.HarvestSummary; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.kernel.setting.SettingInfo; import org.fao.geonet.lib.Lib; import org.fao.geonet.utils.*; @@ -212,12 +213,12 @@ public HarvestResult harvest(Logger log) throws Exception { } //--- harvest metadata and subtemplates from fragments using generic fragment harvester - FragmentHarvester fragmentHarvester = new FragmentHarvester(cancelMonitor, log, context, getFragmentHarvesterParams()); - - if (params.streamFeatures) { - harvestFeatures(wfsQuery, fragmentHarvester); - } else { - harvestResponse(wfsQuery, fragmentHarvester); + try (FragmentHarvester fragmentHarvester = new FragmentHarvester(cancelMonitor, log, context, getFragmentHarvesterParams())) { + if (params.streamFeatures) { + harvestFeatures(wfsQuery, fragmentHarvester); + } else { + harvestResponse(wfsQuery, fragmentHarvester); + } } return result; @@ -334,29 +335,31 @@ public void deleteOrphanedMetadata(Set updatedMetadata) throws Exception if (log.isDebugEnabled()) log.debug(" - Removing orphaned metadata records and fragments after update"); - for (String uuid : localUuids.getUUIDs()) { - try { - String isTemplate = localUuids.getTemplate(uuid); - if (isTemplate.equals("s")) { - Processor.uncacheXLinkUri(metadataGetService + uuid); - } + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter()) { + for (String uuid : localUuids.getUUIDs()) { + try { + String isTemplate = localUuids.getTemplate(uuid); + if (isTemplate.equals("s")) { + Processor.uncacheXLinkUri(metadataGetService + uuid); + } - if (!updatedMetadata.contains(uuid)) { - String id = localUuids.getID(uuid); - metadataManager.deleteMetadata(context, id); + if (!updatedMetadata.contains(uuid)) { + String id = localUuids.getID(uuid); + metadataManager.deleteMetadata(context, id, submitter); - if (isTemplate.equals("s")) { - result.subtemplatesRemoved++; - } else { - result.locallyRemoved++; + if (isTemplate.equals("s")) { + result.subtemplatesRemoved++; + } else { + result.locallyRemoved++; + } } + } catch (CacheException e) { + HarvestError error = new HarvestError(context, e); + this.errors.add(error); + } catch (Exception e) { + HarvestError error = new HarvestError(context, e); + this.errors.add(error); } - } catch (CacheException e) { - HarvestError error = new HarvestError(context, e); - this.errors.add(error); - } catch (Exception e) { - HarvestError error = new HarvestError(context, e); - this.errors.add(error); } } diff --git a/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java b/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java index 4b21f7c61a3..12266a5af55 100644 --- a/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java +++ b/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java @@ -31,7 +31,6 @@ import co.elastic.clients.elasticsearch._types.query_dsl.WrapperQuery; import co.elastic.clients.elasticsearch.cluster.HealthResponse; import co.elastic.clients.elasticsearch.core.*; -import co.elastic.clients.elasticsearch.core.search.Hit; import co.elastic.clients.elasticsearch.indices.AnalyzeRequest; import co.elastic.clients.elasticsearch.indices.AnalyzeResponse; import co.elastic.clients.elasticsearch.indices.analyze.AnalyzeToken; @@ -110,7 +109,7 @@ public ElasticsearchClient getClient() { return client; } - public ElasticsearchAsyncClient getAsynchClient() { + public ElasticsearchAsyncClient getAsyncClient() { return asyncClient; } @@ -208,12 +207,14 @@ public EsRestClient setPassword(String password) { return this; } - public static final String ROUTING_KEY = "101"; - - public BulkResponse bulkRequest(String index, Map docs) throws IOException { + private void checkActivated() { if (!activated) { - throw new IOException("Index not yet activated."); + throw new IllegalStateException("Index not yet activated."); } + } + + public BulkRequest buildIndexBulkRequest(String index, Map docs) { + checkActivated(); BulkRequest.Builder requestBuilder = new BulkRequest.Builder() .index(index) @@ -222,10 +223,7 @@ public BulkResponse bulkRequest(String index, Map docs) throws I JsonpMapper jsonpMapper = client._transport().jsonpMapper(); JsonProvider jsonProvider = jsonpMapper.jsonProvider(); - Iterator> iterator = docs.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - + for (Map.Entry entry : docs.entrySet()) { JsonData jd = JsonData.from(jsonProvider.createParser(new StringReader(entry.getValue())), jsonpMapper); requestBuilder @@ -234,25 +232,32 @@ public BulkResponse bulkRequest(String index, Map docs) throws I .document(jd))); } - BulkRequest request = requestBuilder.build(); + return requestBuilder.build(); + } - try { - return client.bulk(request); - } catch (IOException e) { - e.printStackTrace(); - throw e; + public BulkRequest buildDeleteBulkRequest(String index, List deletionUUIDs) { + checkActivated(); + + BulkRequest.Builder requestBuilder = new BulkRequest.Builder() + .index(index) + .refresh(Refresh.True); + + for (String uuid : deletionUUIDs) { + requestBuilder.operations(op -> op.delete(del -> del.index(index) + .id(uuid))); } + + return requestBuilder.build(); } -// -// public void bulkRequestAsync(Bulk.Builder bulk , JestResultHandler handler) { -// client.executeAsync(bulk.build(), handler); -// -// } -// -// public BulkResult bulkRequestSync(Bulk.Builder bulk) throws IOException { -// return client.execute(bulk.build()); -// } + public DeleteByQueryRequest buildDeleteByQuery(String index, String query) { + checkActivated(); + + return DeleteByQueryRequest.of( + b -> b.index(index) + .q(query) + .refresh(true)); + } /** @@ -358,34 +363,6 @@ public SearchResponse query(String index, Query.Builder queryBuilder, Query.Buil } } - - public String deleteByQuery(String index, String query) throws Exception { - if (!activated) { - return ""; - } - - DeleteByQueryRequest request = DeleteByQueryRequest.of( - b -> b.index(new ArrayList<>(Arrays.asList(index))) - .q(query) - .refresh(true)); - - final DeleteByQueryResponse deleteByQueryResponse = - client.deleteByQuery(request); - - - if (deleteByQueryResponse.deleted() >= 0) { - return String.format("Record removed. %s.", deleteByQueryResponse.deleted()); - } else { - StringBuilder stringBuilder = new StringBuilder(); - - deleteByQueryResponse.failures().forEach(f -> stringBuilder.append(f.toString())); - - throw new IOException(String.format( - "Error during removal. Errors are '%s'.", stringBuilder.toString() - )); - } - } - /** * Get the complete document from the index. * @param id For record index, use UUID. diff --git a/inspire-atom/src/main/java/org/fao/geonet/inspireatom/harvester/InspireAtomHarvester.java b/inspire-atom/src/main/java/org/fao/geonet/inspireatom/harvester/InspireAtomHarvester.java index 7954f45fd67..48fbad12059 100644 --- a/inspire-atom/src/main/java/org/fao/geonet/inspireatom/harvester/InspireAtomHarvester.java +++ b/inspire-atom/src/main/java/org/fao/geonet/inspireatom/harvester/InspireAtomHarvester.java @@ -37,6 +37,7 @@ import org.fao.geonet.kernel.DataManager; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.EsSearchManager; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.InspireAtomFeedRepository; @@ -169,7 +170,7 @@ public final void harvestServiceMetadata(final ServiceContext context, final Str // Removes all atom information from existing metadata. Harvester will reload with updated information localLogger.info("ATOM feed harvest: remove existing metadata feed"); repository.deleteAll(InspireAtomFeedSpecs.hasMetadataId(Integer.parseInt(metadataId))); - dataMan.indexMetadata(Arrays.asList(metadataId)); + dataMan.indexMetadata(metadataId, DirectIndexSubmitter.INSTANCE); // Process service metadata feeds // datasetsInformation stores the dataset information for identifier and namespace for the services feed. @@ -244,7 +245,7 @@ private List processServiceMetadataFeeds(final DataManager data repository.save(inspireAtomFeed); // Index the metadata to store the atom feed information in the index - dataMan.indexMetadata(Arrays.asList(metadataId)); + dataMan.indexMetadata(metadataId, DirectIndexSubmitter.INSTANCE); // Extract datasets information (identifier, namespace) from the service feed: @@ -337,7 +338,7 @@ private void processDatasetsMetadataFeeds(final DataManager dataMan, repository.save(inspireAtomFeed); // Index the metadata to store the atom feed information in the index - dataMan.indexMetadata(Arrays.asList(metadataId)); + dataMan.indexMetadata(metadataId, DirectIndexSubmitter.INSTANCE); result.addContent(new Element("feed").setAttribute("uuid", metadataUuid) .setAttribute("feed", atomUrl).setAttribute("status", "ok")); } catch (Exception ex) { diff --git a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftCreated.java b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftCreated.java index 83d5458e7a6..36a1c74e95c 100644 --- a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftCreated.java +++ b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftCreated.java @@ -23,14 +23,14 @@ package org.fao.geonet.listener.metadata.draft; -import java.util.Arrays; - import org.fao.geonet.constants.Geonet; import org.fao.geonet.domain.AbstractMetadata; import org.fao.geonet.domain.MetadataDraft; import org.fao.geonet.events.md.MetadataDraftAdd; import org.fao.geonet.kernel.datamanager.IMetadataIndexer; import org.fao.geonet.kernel.datamanager.IMetadataUtils; +import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.utils.Log; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationListener; @@ -67,7 +67,7 @@ public void doAfterCommit(MetadataDraftAdd event) { if (!(md instanceof MetadataDraft)) { Log.trace(Geonet.DATA_MANAGER, "Reindexing " + md.getId()); try { - metadataIndexer.indexMetadata(Arrays.asList(String.valueOf(md.getId()))); + metadataIndexer.indexMetadata(String.valueOf(md.getId()), DirectIndexSubmitter.INSTANCE, IndexingMode.full); } catch (Exception e) { Log.error(Geonet.DATA_MANAGER, e, e); } diff --git a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftRemoved.java b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftRemoved.java index bc957e84554..ccff7658346 100644 --- a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftRemoved.java +++ b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftRemoved.java @@ -23,7 +23,6 @@ package org.fao.geonet.listener.metadata.draft; -import java.util.Arrays; import java.util.List; import javax.transaction.Transactional; @@ -35,6 +34,8 @@ import org.fao.geonet.events.md.MetadataDraftRemove; import org.fao.geonet.kernel.datamanager.IMetadataIndexer; import org.fao.geonet.kernel.datamanager.IMetadataUtils; +import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.utils.Log; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -67,7 +68,7 @@ public void doAfterCommit(MetadataDraftRemove event) { if (!(md instanceof MetadataDraft)) { Log.trace(Geonet.DATA_MANAGER, "Reindexing " + md.getId()); try { - metadataIndexer.indexMetadata(Arrays.asList(String.valueOf(md.getId()))); + metadataIndexer.indexMetadata(String.valueOf(md.getId()), DirectIndexSubmitter.INSTANCE, IndexingMode.full); } catch (Exception e) { Log.error(Geonet.DATA_MANAGER, e, e); } diff --git a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftUtilities.java b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftUtilities.java index ce418b4062f..d14733fad32 100644 --- a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftUtilities.java +++ b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftUtilities.java @@ -10,6 +10,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmitter; import org.fao.geonet.repository.*; import org.fao.geonet.repository.specification.MetadataFileUploadSpecs; import org.fao.geonet.repository.specification.MetadataValidationSpecs; @@ -186,7 +187,7 @@ public void removeDraft(MetadataDraft draft) { // --- remove metadata xmlSerializer.delete(String.valueOf(id), ServiceContext.get()); - searchManager.delete(String.format("+id:%d", id )); + searchManager.deleteByUuid(draft.getUuid(), DirectDeletionSubmitter.INSTANCE); } catch (Exception e) { Log.error(Geonet.DATA_MANAGER, "Couldn't cleanup draft " + draft, e); } diff --git a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/RetireRecord.java b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/RetireRecord.java index 98fe1a244ad..2aa26f293a1 100644 --- a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/RetireRecord.java +++ b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/RetireRecord.java @@ -33,6 +33,8 @@ import org.fao.geonet.kernel.datamanager.IMetadataIndexer; import org.fao.geonet.kernel.datamanager.IMetadataOperations; import org.fao.geonet.kernel.datamanager.IMetadataUtils; +import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.utils.Log; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationListener; @@ -40,8 +42,6 @@ import org.springframework.transaction.event.TransactionPhase; import org.springframework.transaction.event.TransactionalEventListener; -import java.util.Arrays; - /** * When a record gets a status change to retired, unpublish it. * @@ -88,7 +88,7 @@ public void doBeforeCommit(MetadataStatusChanged event) { metadataOperations.forceUnsetOperation(context, event.getMd().getId(), ReservedGroup.all.getId(), ReservedOperation.view.getId()); metadataOperations.forceUnsetOperation(context, event.getMd().getId(), ReservedGroup.all.getId(), ReservedOperation.notify.getId()); - metadataIndexer.indexMetadata(Arrays.asList(String.valueOf(event.getMd().getId()))); + metadataIndexer.indexMetadata(String.valueOf(event.getMd().getId()), DirectIndexSubmitter.INSTANCE, IndexingMode.full); } catch (Exception e) { Log.error(Geonet.DATA_MANAGER, "Error upgrading status", e); diff --git a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/UpdateOperations.java b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/UpdateOperations.java index fb6049666c1..57e5a93260d 100644 --- a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/UpdateOperations.java +++ b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/UpdateOperations.java @@ -23,8 +23,6 @@ package org.fao.geonet.listener.metadata.draft; -import java.util.Arrays; - import org.fao.geonet.constants.Geonet; import org.fao.geonet.domain.AbstractMetadata; import org.fao.geonet.domain.Group; @@ -36,6 +34,8 @@ import org.fao.geonet.kernel.datamanager.IMetadataIndexer; import org.fao.geonet.kernel.datamanager.IMetadataOperations; import org.fao.geonet.kernel.datamanager.IMetadataUtils; +import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.repository.GroupRepository; import org.fao.geonet.repository.MetadataDraftRepository; import org.fao.geonet.utils.Log; @@ -121,7 +121,7 @@ public void doAfterCommit(MetadataShare event) { metadataOperations.forceSetOperation(context, draft.getId(), op.getId().getGroupId(), op.getId().getOperationId()); } - metadataIndexer.indexMetadata(Arrays.asList(String.valueOf(draft.getId()))); + metadataIndexer.indexMetadata(String.valueOf(draft.getId()), DirectIndexSubmitter.INSTANCE, IndexingMode.full); } catch (Exception e) { Log.error(Geonet.DATA_MANAGER, "Error cascading operation to draft", e); } diff --git a/services/src/main/java/org/fao/geonet/api/processing/DatabaseProcessUtils.java b/services/src/main/java/org/fao/geonet/api/processing/DatabaseProcessUtils.java index 70808011c86..92dfb0ac1cb 100644 --- a/services/src/main/java/org/fao/geonet/api/processing/DatabaseProcessUtils.java +++ b/services/src/main/java/org/fao/geonet/api/processing/DatabaseProcessUtils.java @@ -26,24 +26,19 @@ import jeeves.server.context.ServiceContext; import org.apache.commons.lang.StringUtils; import org.fao.geonet.api.processing.report.MetadataReplacementProcessingReport; -import org.fao.geonet.api.processing.report.XsltMetadataProcessingReport; import org.fao.geonet.domain.AbstractMetadata; import org.fao.geonet.domain.ISODate; import org.fao.geonet.kernel.AccessManager; import org.fao.geonet.kernel.DataManager; -import org.fao.geonet.kernel.SchemaManager; -import org.fao.geonet.kernel.datamanager.IMetadataManager; -import org.fao.geonet.kernel.datamanager.IMetadataSchemaUtils; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.setting.SettingManager; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.lib.Lib; +import org.fao.geonet.repository.MetadataRepository; import org.fao.geonet.repository.MetadataValidationRepository; import org.fao.geonet.utils.Xml; import org.jdom.Element; -import java.util.Map; - public class DatabaseProcessUtils { /** * Process a metadata with a database SQL query. @@ -109,7 +104,7 @@ public static Element process(ServiceContext context, String id, dataMan.updateMetadata(context, id, wellFormedXml, validate, ufo, language, new ISODate().toString(), updateDateStamp, index ? IndexingMode.full : IndexingMode.none); if (index) { - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, DirectIndexSubmitter.INSTANCE); } } } diff --git a/services/src/main/java/org/fao/geonet/api/processing/XslProcessUtils.java b/services/src/main/java/org/fao/geonet/api/processing/XslProcessUtils.java index e8d52b5dc8b..ef1ee211dc8 100644 --- a/services/src/main/java/org/fao/geonet/api/processing/XslProcessUtils.java +++ b/services/src/main/java/org/fao/geonet/api/processing/XslProcessUtils.java @@ -35,6 +35,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataSchemaUtils; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.lib.Lib; import org.fao.geonet.repository.MetadataValidationRepository; @@ -178,7 +179,7 @@ public void apply(@Nonnull AbstractMetadata entity) { dataMan.updateMetadata(context, id, processedMetadata, validate, ufo, language, new ISODate().toString(), updateDateStamp, IndexingMode.none); if (index) { - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, DirectIndexSubmitter.INSTANCE); } } diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java b/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java index ad25403313c..0ab8dfa62ea 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java @@ -80,6 +80,8 @@ import org.fao.geonet.kernel.mef.MEFLib; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.MetadataDraftRepository; @@ -241,10 +243,8 @@ public void deleteRecord( store.delResources(context, metadata.getUuid(), approved); RecordDeletedEvent recordDeletedEvent = triggerDeletionEvent(request, metadata.getId() + ""); - metadataManager.deleteMetadata(context, metadata.getId() + ""); + metadataManager.deleteMetadata(context, metadata.getId() + "", DirectDeletionSubmitter.INSTANCE); recordDeletedEvent.publish(ApplicationContextHolder.get()); - - dataManager.forceIndexChanges(); } @io.swagger.v3.oas.annotations.Operation(summary = "Delete one or more records", description ="User MUST be able to edit the record to delete it. " @@ -302,7 +302,7 @@ public SimpleMetadataProcessingReport deleteRecords( store.delResources(context, metadata.getUuid()); RecordDeletedEvent recordDeletedEvent = triggerDeletionEvent(request, String.valueOf(metadata.getId())); - metadataManager.deleteMetadata(context, String.valueOf(metadata.getId())); + metadataManager.deleteMetadata(context, String.valueOf(metadata.getId()), DirectDeletionSubmitter.INSTANCE); recordDeletedEvent.publish(ApplicationContextHolder.get()); report.incrementProcessedRecords(); @@ -1022,7 +1022,7 @@ private Pair loadRecord(MetadataType metadataType, Element xmlE metadataValidator.doValidate(metadata, context.getLanguage()); } - dataManager.indexMetadata(id.get(0), true); + dataManager.indexMetadata(id.get(0), DirectIndexSubmitter.INSTANCE); return Pair.read(Integer.valueOf(id.get(0)), uuid); } diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataSampleApi.java b/services/src/main/java/org/fao/geonet/api/records/MetadataSampleApi.java index fd1396387be..2cd757b9198 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataSampleApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataSampleApi.java @@ -33,7 +33,6 @@ import jeeves.services.ReadWriteController; import org.apache.commons.lang.StringUtils; import org.fao.geonet.ApplicationContextHolder; -import org.fao.geonet.api.API; import org.fao.geonet.api.ApiParams; import org.fao.geonet.api.ApiUtils; import org.fao.geonet.api.processing.report.SimpleMetadataProcessingReport; @@ -50,6 +49,7 @@ import org.fao.geonet.kernel.UpdateDatestamp; import org.fao.geonet.kernel.mef.MEFLib; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.utils.Log; import org.fao.geonet.utils.Xml; @@ -290,7 +290,7 @@ SimpleMetadataProcessingReport addTemplates( if (MetadataType.lookup(isTemplate) == MetadataType.TEMPLATE) { xml = dataManager.setUUID(schemaName, uuid, xml); } - dataManager.insertMetadata(context, metadata, xml, IndexingMode.full, true, UpdateDatestamp.NO, false, true); + dataManager.insertMetadata(context, metadata, xml, IndexingMode.full, true, UpdateDatestamp.NO, false, DirectIndexSubmitter.INSTANCE); report.addMetadataInfos(metadata, String.format( "Template for schema '%s' with UUID '%s' added.", diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataSharingApi.java b/services/src/main/java/org/fao/geonet/api/records/MetadataSharingApi.java index 5967992f740..d097928aa7f 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataSharingApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataSharingApi.java @@ -55,6 +55,7 @@ import org.fao.geonet.kernel.DataManager; import org.fao.geonet.kernel.datamanager.*; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.languages.FeedbackLanguages; @@ -818,7 +819,7 @@ public void setRecordGroup( metadata.getSourceInfo().setGroupOwner(groupIdentifier); metadataManager.save(metadata); - dataManager.indexMetadata(String.valueOf(metadata.getId()), true); + dataManager.indexMetadata(String.valueOf(metadata.getId()), DirectIndexSubmitter.INSTANCE); new RecordGroupOwnerChangeEvent(metadata.getId(), ApiUtils.getUserSession(request.getSession()).getUserIdAsInt(), @@ -1153,7 +1154,7 @@ private void checkCanPublishToAllGroup(ServiceContext context, ResourceBundle me if (!hasValidation) { validator.doValidate(metadata, context.getLanguage()); - metadataIndexer.indexMetadata(metadata.getId() + "", true, IndexingMode.full); + metadataIndexer.indexMetadata(metadata.getId() + "", DirectIndexSubmitter.INSTANCE, IndexingMode.full); } boolean isInvalid = @@ -1217,7 +1218,7 @@ private void shareMetadataWithReservedGroup(String metadataUuid, boolean publish setOperations(sharing, dataManager, context, appContext, metadata, operationMap, privileges, ApiUtils.getUserSession(session).getUserIdAsInt(), true, null, request, metadataListToNotifyPublication, notifyByEmail); - metadataIndexer.indexMetadata(String.valueOf(metadata.getId()), true, IndexingMode.full); + metadataIndexer.indexMetadata(String.valueOf(metadata.getId()), DirectIndexSubmitter.INSTANCE, IndexingMode.full); java.util.Optional publicationOption = publicationConfig.getPublicationOptionConfiguration(publicationType); if (publicationOption.isPresent() && diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataUtils.java b/services/src/main/java/org/fao/geonet/api/records/MetadataUtils.java index 09a9794a489..cd5c0971397 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataUtils.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataUtils.java @@ -53,6 +53,7 @@ import org.fao.geonet.kernel.schema.AssociatedResourcesSchemaPlugin; import org.fao.geonet.kernel.schema.SchemaPlugin; import org.fao.geonet.kernel.search.EsSearchManager; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingInfo; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.lib.Lib; @@ -766,7 +767,7 @@ public static boolean retrieveMetadataValidationStatus(AbstractMetadata metadata if (!hasValidation) { validator.doValidate(metadata, context.getLanguage()); - dataManager.indexMetadata(metadata.getId() + "", true); + dataManager.indexMetadata(metadata.getId() + "", DirectIndexSubmitter.INSTANCE); } boolean isInvalid = diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataValidateApi.java b/services/src/main/java/org/fao/geonet/api/records/MetadataValidateApi.java index fc7e699fc24..6f9469cde66 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataValidateApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataValidateApi.java @@ -31,7 +31,6 @@ import jeeves.server.context.ServiceContext; import jeeves.services.ReadWriteController; import org.fao.geonet.ApplicationContextHolder; -import org.fao.geonet.api.API; import org.fao.geonet.api.ApiParams; import org.fao.geonet.api.ApiUtils; import org.fao.geonet.api.records.editing.AjaxEditUtils; @@ -45,6 +44,7 @@ import org.fao.geonet.kernel.DataManager; import org.fao.geonet.kernel.GeonetworkDataDirectory; import org.fao.geonet.kernel.schema.MetadataSchema; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.repository.MetadataValidationRepository; import org.fao.geonet.repository.SchematronRepository; import org.fao.geonet.utils.IO; @@ -189,7 +189,7 @@ Reports validateRecord( .setStatus(isvalid ? MetadataValidationStatus.VALID : MetadataValidationStatus.INVALID) .setRequired(true).setNumTests(0).setNumFailures(0); this.metadataValidationRepository.save(metadataValidation); - dataManager.indexMetadata(("" + metadata.getId()), true); + dataManager.indexMetadata(("" + metadata.getId()), DirectIndexSubmitter.INSTANCE); new RecordValidationTriggeredEvent(metadata.getId(), ApiUtils.getUserSession(request.getSession()).getUserIdAsInt(), metadataValidation.getStatus().getCode()).publish(appContext); diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataWorkflowApi.java b/services/src/main/java/org/fao/geonet/api/records/MetadataWorkflowApi.java index dd72b20e8fe..5d32c63037d 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataWorkflowApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataWorkflowApi.java @@ -58,6 +58,7 @@ import org.fao.geonet.kernel.metadata.StatusChangeType; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.languages.FeedbackLanguages; @@ -517,7 +518,7 @@ public Map setStatus(@Parameter(description = API_PAR if (statusUpdate.get(metadata.getId()) == StatusChangeType.UPDATED) { //--- reindex metadata - metadataIndexer.indexMetadata(String.valueOf(metadata.getId()), true, IndexingMode.full); + metadataIndexer.indexMetadata(String.valueOf(metadata.getId()), DirectIndexSubmitter.INSTANCE, IndexingMode.full); // Reindex the metadata table record to update the field _statusWorkflow that contains the composite // status of the published and draft versions @@ -526,7 +527,7 @@ public Map setStatus(@Parameter(description = API_PAR if (metadataApproved != null) { metadataIdApproved = metadataApproved.getId(); - metadataIndexer.indexMetadata(String.valueOf(metadataApproved.getId()), true, IndexingMode.full); + metadataIndexer.indexMetadata(String.valueOf(metadataApproved.getId()), DirectIndexSubmitter.INSTANCE, IndexingMode.full); } } } @@ -945,7 +946,7 @@ public void restoreAtStatusSave( recoveredMetadataId = reloadRecord(element, metadataManager, httpSession, request); } - metadataIndexer.indexMetadata(String.valueOf(recoveredMetadataId), true, IndexingMode.full); + metadataIndexer.indexMetadata(String.valueOf(recoveredMetadataId), DirectIndexSubmitter.INSTANCE, IndexingMode.full); UserSession session = ApiUtils.getUserSession(request.getSession()); if (session != null) { diff --git a/services/src/main/java/org/fao/geonet/api/records/editing/AjaxEditUtils.java b/services/src/main/java/org/fao/geonet/api/records/editing/AjaxEditUtils.java index cf06d7c9e34..18c1170a789 100644 --- a/services/src/main/java/org/fao/geonet/api/records/editing/AjaxEditUtils.java +++ b/services/src/main/java/org/fao/geonet/api/records/editing/AjaxEditUtils.java @@ -37,6 +37,7 @@ import org.fao.geonet.kernel.UpdateDatestamp; import org.fao.geonet.kernel.schema.MetadataSchema; import org.fao.geonet.kernel.schema.SchemaPlugin; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.lib.Lib; import org.fao.geonet.schema.iso19139.ISO19139Namespaces; import org.fao.geonet.utils.Log; @@ -695,7 +696,7 @@ public synchronized boolean addAttribute(String id, String ref, String name, Str String changeDate = null; xmlSerializer.update(id, md, changeDate, false, null, context); - dataManager.indexMetadata(id, true); + dataManager.indexMetadata(id, DirectIndexSubmitter.INSTANCE); return true; } @@ -740,7 +741,7 @@ public synchronized boolean deleteAttribute(String id, String ref, String name, String changeDate = null; xmlSerializer.update(id, md, changeDate, false, null, context); - dataManager.indexMetadata(id, true); + dataManager.indexMetadata(id, DirectIndexSubmitter.INSTANCE); return true; } diff --git a/services/src/main/java/org/fao/geonet/api/records/editing/MetadataEditingApi.java b/services/src/main/java/org/fao/geonet/api/records/editing/MetadataEditingApi.java index 16aaee8d1d5..7a0383f4d66 100644 --- a/services/src/main/java/org/fao/geonet/api/records/editing/MetadataEditingApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/editing/MetadataEditingApi.java @@ -54,6 +54,7 @@ import org.fao.geonet.kernel.metadata.StatusActions; import org.fao.geonet.kernel.metadata.StatusActionsFactory; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.*; @@ -485,7 +486,7 @@ public void saveEdits( if (reindex) { Log.trace(Geonet.DATA_MANAGER, " > Reindexing record"); - metadataIndexer.indexMetadata(id, true, IndexingMode.full); + metadataIndexer.indexMetadata(id, DirectIndexSubmitter.INSTANCE, IndexingMode.full); } // Reindex the metadata table record to update the field _statusWorkflow that contains the composite @@ -494,7 +495,7 @@ public void saveEdits( Metadata metadataApproved = metadataRepository.findOneByUuid(metadata.getUuid()); if (metadataApproved != null) { - metadataIndexer.indexMetadata(String.valueOf(metadataApproved.getId()), true, IndexingMode.full); + metadataIndexer.indexMetadata(String.valueOf(metadataApproved.getId()), DirectIndexSubmitter.INSTANCE, IndexingMode.full); } } diff --git a/services/src/main/java/org/fao/geonet/api/registries/DirectoryUtils.java b/services/src/main/java/org/fao/geonet/api/registries/DirectoryUtils.java index a011b58b68d..96e7c6a1225 100644 --- a/services/src/main/java/org/fao/geonet/api/registries/DirectoryUtils.java +++ b/services/src/main/java/org/fao/geonet/api/registries/DirectoryUtils.java @@ -3,7 +3,6 @@ import co.elastic.clients.elasticsearch.core.SearchResponse; import co.elastic.clients.elasticsearch.core.search.Hit; import com.google.common.collect.Table; -import jeeves.server.ServiceConfig; import jeeves.server.context.ServiceContext; import jeeves.xlink.XLink; import org.apache.commons.lang.StringUtils; @@ -13,6 +12,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.repository.MetadataRepository; import org.fao.geonet.util.Sha1Encoder; import org.fao.geonet.utils.Log; @@ -44,7 +44,7 @@ public static Map saveEntries(ServiceContext context, entries.rowKeySet().iterator(); AbstractMetadata record = collectResults.getRecord(); boolean validate = false, ufo = false, - notify = false, publicForGroup = true, refreshReaders = false; + notify = false, publicForGroup = true; Map errors = new HashMap<>(); while (entriesIterator.hasNext()) { @@ -73,7 +73,7 @@ public static Map saveEntries(ServiceContext context, (Element) entry.clone(), IndexingMode.none, ufo, UpdateDatestamp.NO, - publicForGroup, refreshReaders); + publicForGroup, DirectIndexSubmitter.INSTANCE); collectResults.getEntryIdentifiers().put( uuid, subtemplate.getId()); diff --git a/services/src/main/java/org/fao/geonet/api/selections/UserSelectionsApi.java b/services/src/main/java/org/fao/geonet/api/selections/UserSelectionsApi.java index 595ec851698..6932db2edd8 100644 --- a/services/src/main/java/org/fao/geonet/api/selections/UserSelectionsApi.java +++ b/services/src/main/java/org/fao/geonet/api/selections/UserSelectionsApi.java @@ -37,6 +37,8 @@ import org.fao.geonet.kernel.datamanager.IMetadataIndexer; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmitter; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.repository.LanguageRepository; import org.fao.geonet.repository.SelectionRepository; import org.fao.geonet.repository.UserGroupRepository; @@ -325,7 +327,7 @@ ResponseEntity addToUserSelection( UserSavedSelection e = new UserSavedSelection(selection.get(), user.get(), u); try { umsRepository.save(e); - metadataIndexer.indexMetadata(metadataRepository.getMetadataId(u), true, IndexingMode.full); + metadataIndexer.indexMetadata(metadataRepository.getMetadataId(u), DirectIndexSubmitter.INSTANCE, IndexingMode.full); } catch (Exception e1) { Log.error(API.LOG_MODULE_NAME, "UserSelectionsApi - addToUserSelection: " + e1.getMessage(), e1); } @@ -384,13 +386,15 @@ ResponseEntity deleteFromUserSelection( if (uuid == null || uuid.length == 0) { umsRepository.deleteAllBySelectionAndUser(selectionIdentifier, userIdentifier); } else { - for (String u : uuid) { - UserSavedSelectionId e = new UserSavedSelectionId() - .setSelectionId(selectionIdentifier) - .setUserId(userIdentifier) - .setMetadataUuid(u); - umsRepository.deleteById(e); - metadataIndexer.indexMetadata(metadataRepository.getMetadataId(u), true, IndexingMode.full); + try (BatchingIndexSubmitter batchingIndexSubmittor = new BatchingIndexSubmitter(uuid.length)) { + for (String u : uuid) { + UserSavedSelectionId e = new UserSavedSelectionId() + .setSelectionId(selectionIdentifier) + .setUserId(userIdentifier) + .setMetadataUuid(u); + umsRepository.deleteById(e); + metadataIndexer.indexMetadata(metadataRepository.getMetadataId(u), batchingIndexSubmittor, IndexingMode.full); + } } } diff --git a/services/src/main/java/org/fao/geonet/api/site/SiteApi.java b/services/src/main/java/org/fao/geonet/api/site/SiteApi.java index 01882b580ed..e024af209fe 100644 --- a/services/src/main/java/org/fao/geonet/api/site/SiteApi.java +++ b/services/src/main/java/org/fao/geonet/api/site/SiteApi.java @@ -61,6 +61,7 @@ import org.fao.geonet.kernel.datamanager.base.BaseMetadataManager; import org.fao.geonet.kernel.harvest.HarvestManager; import org.fao.geonet.kernel.search.EsSearchManager; +import org.fao.geonet.kernel.search.index.BatchOpsMetadataReindexer; import org.fao.geonet.kernel.setting.SettingInfo; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; @@ -553,7 +554,7 @@ public boolean isIndexing( HttpServletRequest request ) throws Exception { ApiUtils.createServiceContext(request); - return ApplicationContextHolder.get().getBean(DataManager.class).isIndexing(); + return BatchOpsMetadataReindexer.isIndexing(); } @io.swagger.v3.oas.annotations.Operation( @@ -570,10 +571,6 @@ public HttpEntity indexSite( required = false) @RequestParam(required = false, defaultValue = "true") boolean reset, - @Parameter(description = "Asynchronous mode (only on all records. ie. no selection bucket)", - required = false) - @RequestParam(required = false, defaultValue = "false") - boolean asynchronous, @Parameter(description = "Index. By default only remove record index.", required = false) @RequestParam(required = false, defaultValue = "records") @@ -589,8 +586,7 @@ public HttpEntity indexSite( ) throws Exception { ServiceContext context = ApiUtils.createServiceContext(request); EsSearchManager searchMan = ApplicationContextHolder.get().getBean(EsSearchManager.class); - DataManager dataManager = ApplicationContextHolder.get().getBean(DataManager.class); - boolean isIndexing = dataManager.isIndexing(); + boolean isIndexing = BatchOpsMetadataReindexer.isIndexing(); if (isIndexing) { throw new NotAllowedException( @@ -606,7 +602,7 @@ public HttpEntity indexSite( if (StringUtils.isEmpty(bucket)) { BaseMetadataManager metadataManager = ApplicationContextHolder.get().getBean(BaseMetadataManager.class); - metadataManager.synchronizeDbWithIndex(context, false, asynchronous); + metadataManager.synchronizeDbWithIndex(context); } else { searchMan.rebuildIndex(context, false, bucket); } @@ -614,22 +610,6 @@ public HttpEntity indexSite( return new HttpEntity<>(HttpStatus.CREATED); } - @io.swagger.v3.oas.annotations.Operation( - summary = "Index commit", - description = "") - @RequestMapping( - path = "/index/commit", - produces = MediaType.APPLICATION_JSON_VALUE, - method = RequestMethod.GET) - @ResponseStatus(HttpStatus.OK) - @PreAuthorize("hasAuthority('Administrator')") - public void indexCommit( - HttpServletRequest request - ) throws Exception { - EsSearchManager searchMan = ApplicationContextHolder.get().getBean(EsSearchManager.class); - searchMan.forceIndexChanges(); - } - @io.swagger.v3.oas.annotations.Operation( summary = "Index status", @@ -676,24 +656,6 @@ public Map indexAndDbSynchronizationStatus( } - @io.swagger.v3.oas.annotations.Operation( - summary = "Force to commit pending documents in index.", - description = "May be used when indexing task is hanging.") - @PutMapping( - path = "/index/commit") - @ApiResponses(value = { - @ApiResponse(responseCode = "201", description = "Changes committed.") - }) - @ResponseStatus(HttpStatus.CREATED) - @PreAuthorize("hasAuthority('Administrator')") - public void commitIndexChanges( - ) throws Exception { - ApplicationContextHolder.get() - .getBean(EsSearchManager.class) - .forceIndexChanges(); - } - - @io.swagger.v3.oas.annotations.Operation( summary = "Get build details", description = "To know when and how this version of the application was built.") diff --git a/services/src/main/java/org/fao/geonet/inspire/validator/InspireValidationRunnable.java b/services/src/main/java/org/fao/geonet/inspire/validator/InspireValidationRunnable.java index ab8d64557b6..2b558dd0295 100644 --- a/services/src/main/java/org/fao/geonet/inspire/validator/InspireValidationRunnable.java +++ b/services/src/main/java/org/fao/geonet/inspire/validator/InspireValidationRunnable.java @@ -26,17 +26,14 @@ import jeeves.transaction.TransactionManager; import jeeves.transaction.TransactionTask; import org.fao.geonet.ApplicationContextHolder; -import org.fao.geonet.inspire.validator.InspireValidatorUtils; import org.fao.geonet.domain.MetadataValidation; import org.fao.geonet.domain.MetadataValidationId; import org.fao.geonet.domain.MetadataValidationStatus; import org.fao.geonet.kernel.DataManager; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.repository.MetadataValidationRepository; import org.springframework.transaction.TransactionStatus; -import java.util.ArrayList; -import java.util.Arrays; - import static jeeves.transaction.TransactionManager.CommitBehavior.ALWAYS_COMMIT; import static jeeves.transaction.TransactionManager.TransactionRequirement.CREATE_NEW; @@ -91,7 +88,7 @@ public Object doInTransaction(TransactionStatus transaction) throws Throwable { DataManager dataManager = ApplicationContextHolder.get().getBean(DataManager.class); - dataManager.indexMetadata(new ArrayList<>(Arrays.asList(mdId + ""))); + dataManager.indexMetadata(mdId + "", DirectIndexSubmitter.INSTANCE); return null; } diff --git a/services/src/main/java/org/fao/geonet/inspire/validator/MInspireEtfValidateProcess.java b/services/src/main/java/org/fao/geonet/inspire/validator/MInspireEtfValidateProcess.java index 07c1faf1c71..61036c007c8 100644 --- a/services/src/main/java/org/fao/geonet/inspire/validator/MInspireEtfValidateProcess.java +++ b/services/src/main/java/org/fao/geonet/inspire/validator/MInspireEtfValidateProcess.java @@ -16,6 +16,7 @@ import org.fao.geonet.kernel.SchemaManager; import org.fao.geonet.kernel.datamanager.IMetadataSchemaUtils; import org.fao.geonet.kernel.datamanager.IMetadataUtils; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.repository.MetadataValidationRepository; import org.fao.geonet.repository.SourceRepository; @@ -33,8 +34,6 @@ import javax.management.ObjectName; import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Map; import java.util.Set; @@ -263,7 +262,7 @@ public Object doInTransaction(TransactionStatus transaction) throws Throwable { } if (reindexMetadata) { - dataManager.indexMetadata(new ArrayList<>(Arrays.asList(metadataRecord.getId() + ""))); + dataManager.indexMetadata(metadataRecord.getId() + "", DirectIndexSubmitter.INSTANCE); } } catch (Exception ex) { diff --git a/services/src/main/java/org/fao/geonet/services/main/Info.java b/services/src/main/java/org/fao/geonet/services/main/Info.java index 6cf34804bc3..cc591635cf3 100644 --- a/services/src/main/java/org/fao/geonet/services/main/Info.java +++ b/services/src/main/java/org/fao/geonet/services/main/Info.java @@ -34,9 +34,9 @@ import org.fao.geonet.constants.Geonet; import org.fao.geonet.domain.*; import org.fao.geonet.exceptions.BadParameterEx; -import org.fao.geonet.kernel.DataManager; import org.fao.geonet.kernel.SchemaManager; import org.fao.geonet.kernel.region.RegionsDAO; +import org.fao.geonet.kernel.search.index.BatchOpsMetadataReindexer; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.lib.Lib; @@ -253,7 +253,7 @@ public Element exec(Element inParams, ServiceContext context) throws Exception { */ private Element getIndex(GeonetContext gc) { Element isIndexing = new Element(INDEX); - isIndexing.setText(Boolean.toString(gc.getBean(DataManager.class).isIndexing())); + isIndexing.setText(Boolean.toString(BatchOpsMetadataReindexer.isIndexing())); return isIndexing; } diff --git a/services/src/test/java/org/fao/geonet/api/links/LinksApiTest.java b/services/src/test/java/org/fao/geonet/api/links/LinksApiTest.java index 508f5cca808..bb95bebfe8e 100644 --- a/services/src/test/java/org/fao/geonet/api/links/LinksApiTest.java +++ b/services/src/test/java/org/fao/geonet/api/links/LinksApiTest.java @@ -98,7 +98,7 @@ public void getLinks() throws Exception { private void createTestData() throws Exception { loginAsAdmin(context); - this.md = injectMetadataInDb(getSampleMetadataXml(), context, true); + this.md = injectMetadataInDb(getSampleMetadataXml(), context); Element xmlData = this.md.getXmlData(false); urlAnalyzer.processMetadata(xmlData, this.md); diff --git a/services/src/test/java/org/fao/geonet/api/records/MetadataApiTest.java b/services/src/test/java/org/fao/geonet/api/records/MetadataApiTest.java index dd9f3880916..82006ecc5c3 100644 --- a/services/src/test/java/org/fao/geonet/api/records/MetadataApiTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/MetadataApiTest.java @@ -80,7 +80,7 @@ public void setUp() throws Exception { private void createTestData() throws Exception { loginAsAdmin(context); - AbstractMetadata metadata = injectMetadataInDb(getSampleMetadataXml(), context, true); + AbstractMetadata metadata = injectMetadataInDb(getSampleMetadataXml(), context); id = metadata.getId(); uuid = metadata.getUuid(); } diff --git a/services/src/test/java/org/fao/geonet/api/records/MetadataValidateApiTest.java b/services/src/test/java/org/fao/geonet/api/records/MetadataValidateApiTest.java index 4f8afb52795..b24fb3bbfb4 100644 --- a/services/src/test/java/org/fao/geonet/api/records/MetadataValidateApiTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/MetadataValidateApiTest.java @@ -24,6 +24,7 @@ import org.fao.geonet.kernel.SchemaManager; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.repository.MetadataRepository; import org.fao.geonet.repository.MetadataValidationRepository; import org.fao.geonet.repository.SourceRepository; @@ -208,9 +209,9 @@ private AbstractMetadata subTemplateOnLineResourceDbInsert(MetadataType type) th false, NO, false, - false); + DirectIndexSubmitter.INSTANCE); - dataManager.indexMetadata("" + dbInsertedMetadata.getId(), true); + dataManager.indexMetadata("" + dbInsertedMetadata.getId(), DirectIndexSubmitter.INSTANCE); assertEquals(1, countTemplateIndexed(dbInsertedMetadata.getUuid(), false, type == MetadataType.SUB_TEMPLATE ? "s" : "n")); return dbInsertedMetadata; } diff --git a/services/src/test/java/org/fao/geonet/api/records/MetadataWorkflowApiTest.java b/services/src/test/java/org/fao/geonet/api/records/MetadataWorkflowApiTest.java index a56815862c2..87ec40f129a 100644 --- a/services/src/test/java/org/fao/geonet/api/records/MetadataWorkflowApiTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/MetadataWorkflowApiTest.java @@ -56,10 +56,10 @@ public void setUp() throws Exception { settingManager.setValue(Settings.METADATA_WORKFLOW_ENABLE, true); settingManager.setValue(Settings.METADATA_WORKFLOW_DRAFT_WHEN_IN_GROUP, ".*"); - AbstractMetadata metadata = injectMetadataInDbDoNotRefreshHeader(getSampleMetadataXml(), context); + AbstractMetadata metadata = injectMetadataInDb(getSampleMetadataXml(), context); injectStatusForMetadata(metadata, DRAFT); uuid = metadata.getUuid(); - AbstractMetadata metadata2 = injectMetadataInDbDoNotRefreshHeader(getSampleMetadataXml(), context); + AbstractMetadata metadata2 = injectMetadataInDb(getSampleMetadataXml(), context); injectStatusForMetadata(metadata2, APPROVED); uuid2 = metadata2.getUuid(); } diff --git a/services/src/test/java/org/fao/geonet/api/records/extent/MetadataExtentApiTest.java b/services/src/test/java/org/fao/geonet/api/records/extent/MetadataExtentApiTest.java index 20ab40561e6..641f477a99f 100644 --- a/services/src/test/java/org/fao/geonet/api/records/extent/MetadataExtentApiTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/extent/MetadataExtentApiTest.java @@ -24,7 +24,6 @@ package org.fao.geonet.api.records.extent; import jeeves.server.context.ServiceContext; -import org.fao.geonet.domain.AbstractMetadata; import org.fao.geonet.domain.ISODate; import org.fao.geonet.domain.Metadata; import org.fao.geonet.domain.MetadataType; @@ -32,6 +31,7 @@ import org.fao.geonet.kernel.SchemaManager; import org.fao.geonet.kernel.UpdateDatestamp; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.repository.SourceRepository; import org.fao.geonet.services.AbstractServiceIntegrationTest; import org.fao.geonet.utils.Xml; @@ -55,14 +55,11 @@ import java.io.File; import java.io.IOException; import java.net.URL; -import java.util.Arrays; import java.util.Calendar; import java.util.GregorianCalendar; import java.util.TimeZone; import java.util.UUID; -import static org.fao.geonet.schema.iso19139.ISO19139Namespaces.GCO; -import static org.fao.geonet.schema.iso19139.ISO19139Namespaces.GMD; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; @@ -339,7 +336,7 @@ private String createMdFromXmlRessources(Element sampleMetadataXml) throws Excep .setHarvested(false); dataManager.insertMetadata(context, metadata, sampleMetadataXml, IndexingMode.full, false, UpdateDatestamp.NO, - false, false).getId(); + false, DirectIndexSubmitter.INSTANCE).getId(); return uuid; } diff --git a/services/src/test/java/org/fao/geonet/api/records/formatters/AbstractFormatterTest.java b/services/src/test/java/org/fao/geonet/api/records/formatters/AbstractFormatterTest.java index 285b4e29f05..41e96825bc5 100644 --- a/services/src/test/java/org/fao/geonet/api/records/formatters/AbstractFormatterTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/formatters/AbstractFormatterTest.java @@ -34,6 +34,7 @@ import org.fao.geonet.kernel.SchemaManager; import org.fao.geonet.kernel.UpdateDatestamp; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.languages.IsoLanguagesMapper; import org.fao.geonet.repository.MetadataRepository; @@ -97,7 +98,7 @@ public void setUp() throws Exception { metadata.setHarvestInfo(harvestInfo); metadata.setData(xml); this.id = dataManager.insertMetadata(serviceContext, metadata, metadata.getXmlData(false), IndexingMode.full, false, - UpdateDatestamp.NO, false, false).getId(); + UpdateDatestamp.NO, false, DirectIndexSubmitter.INSTANCE).getId(); } public boolean isHarvested() { diff --git a/services/src/test/java/org/fao/geonet/api/records/formatters/AlternateLogoForPdfExportTest.java b/services/src/test/java/org/fao/geonet/api/records/formatters/AlternateLogoForPdfExportTest.java index be863787a89..0209e4af23e 100644 --- a/services/src/test/java/org/fao/geonet/api/records/formatters/AlternateLogoForPdfExportTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/formatters/AlternateLogoForPdfExportTest.java @@ -53,7 +53,7 @@ public void initSiteId() { public void createTestData() throws Exception { context = createServiceContext(); loginAsAdmin(context); - metadata = injectMetadataInDbDoNotRefreshHeader(getSampleISO19139MetadataXml(), context); + metadata = injectMetadataInDb(getSampleISO19139MetadataXml(), context); } @Before diff --git a/services/src/test/java/org/fao/geonet/api/records/formatters/FormatterApiIntegrationTest.java b/services/src/test/java/org/fao/geonet/api/records/formatters/FormatterApiIntegrationTest.java index 60bf0c6a49f..813073d5f62 100644 --- a/services/src/test/java/org/fao/geonet/api/records/formatters/FormatterApiIntegrationTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/formatters/FormatterApiIntegrationTest.java @@ -75,7 +75,7 @@ public class FormatterApiIntegrationTest extends AbstractServiceIntegrationTest public void setUp() throws Exception { serviceContext = createServiceContext(); loginAsAdmin(serviceContext); - AbstractMetadata metadata = injectMetadataInDbDoNotRefreshHeader(getSampleMetadataXml(), serviceContext); + AbstractMetadata metadata = injectMetadataInDb(getSampleMetadataXml(), serviceContext); id = metadata.getId(); schema = metadata.getDataInfo().getSchemaId(); } diff --git a/services/src/test/java/org/fao/geonet/api/records/formatters/FormatterApiTest.java b/services/src/test/java/org/fao/geonet/api/records/formatters/FormatterApiTest.java index 4d0f3c79c31..a9c1b8293c6 100644 --- a/services/src/test/java/org/fao/geonet/api/records/formatters/FormatterApiTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/formatters/FormatterApiTest.java @@ -298,7 +298,7 @@ private void createTestData() throws Exception { } private void loadFile(String key, Element sampleMetadataXml) throws Exception { - AbstractMetadata metadata = injectMetadataInDbDoNotRefreshHeader(sampleMetadataXml, context); + AbstractMetadata metadata = injectMetadataInDb(sampleMetadataXml, context); testDataUuidBySchema.put(key, metadata.getUuid()); } } diff --git a/services/src/test/java/org/fao/geonet/api/records/formatters/cache/FormatterCacheIntegrationTest.java b/services/src/test/java/org/fao/geonet/api/records/formatters/cache/FormatterCacheIntegrationTest.java index 387a1886882..aded0661464 100644 --- a/services/src/test/java/org/fao/geonet/api/records/formatters/cache/FormatterCacheIntegrationTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/formatters/cache/FormatterCacheIntegrationTest.java @@ -32,6 +32,7 @@ import org.fao.geonet.domain.ReservedOperation; import org.fao.geonet.kernel.datamanager.IMetadataManager; import org.fao.geonet.kernel.mef.MEFLibIntegrationTest.ImportMetadata; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmitter; import org.fao.geonet.repository.OperationAllowedRepository; import org.fao.geonet.repository.specification.OperationAllowedSpecs; import org.fao.geonet.services.AbstractServiceIntegrationTest; @@ -134,7 +135,7 @@ public void testUpdatesAfterMetadataDeleted() throws Exception { final Key key = new Key(Integer.parseInt(metadataId), "eng", FormatType.html, "full_view", true, FormatterWidth._100); formatterCache.get(key, new ChangeDateValidator(changeDate), new TestLoader("result", changeDate, true), true); - metadataManager.deleteMetadata(createServiceContext(), metadataId); + metadataManager.deleteMetadata(createServiceContext(), metadataId, DirectDeletionSubmitter.INSTANCE); entityManager.flush(); assertNull(formatterCache.getPublished(key)); diff --git a/services/src/test/java/org/fao/geonet/api/selections/UserSelectionsApiTest.java b/services/src/test/java/org/fao/geonet/api/selections/UserSelectionsApiTest.java index 9cdd6ed90d9..6abc191bbb1 100644 --- a/services/src/test/java/org/fao/geonet/api/selections/UserSelectionsApiTest.java +++ b/services/src/test/java/org/fao/geonet/api/selections/UserSelectionsApiTest.java @@ -32,13 +32,14 @@ import org.fao.geonet.kernel.datamanager.base.BaseMetadataIndexer; import org.fao.geonet.kernel.mef.MEFLibIntegrationTest; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; +import org.fao.geonet.kernel.search.submission.IIndexSubmitter; import org.fao.geonet.repository.MetadataRepository; import org.fao.geonet.repository.SelectionRepository; import org.fao.geonet.services.AbstractServiceIntegrationTest; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; -import org.mockito.Spy; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; import org.springframework.mock.web.MockHttpSession; @@ -51,8 +52,7 @@ import java.util.HashMap; import java.util.Map; -import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -229,7 +229,7 @@ public void addDeleteSelection() throws Exception { .accept(MediaType.parseMediaType("application/json"))) .andExpect(status().isCreated()); - verify(this.metadataIndexerSpy, times(1)).indexMetadata(eq(metadataId), any(Boolean.class), eq(IndexingMode.full)); + verify(this.metadataIndexerSpy, times(1)).indexMetadata(eq(metadataId), eq(DirectIndexSubmitter.INSTANCE), eq(IndexingMode.full)); this.mockMvc.perform(get("/srv/api/userselections/" + createdSelection.getId() + "/1") .session(this.mockHttpSession) @@ -244,7 +244,7 @@ public void addDeleteSelection() throws Exception { .accept(MediaType.parseMediaType("application/json"))) .andExpect(status().isNoContent()); - verify(this.metadataIndexerSpy, times(2)).indexMetadata(eq(metadataId), any(Boolean.class), eq(IndexingMode.full)); + verify(this.metadataIndexerSpy, times(2)).indexMetadata(eq(metadataId), any(IIndexSubmitter.class), eq(IndexingMode.full)); // Delete this.mockMvc.perform(delete("/srv/api/userselections/" + createdSelection.getId()) diff --git a/services/src/test/java/org/fao/geonet/services/metadata/BatchOpsMetadatReindexerTest.java b/services/src/test/java/org/fao/geonet/services/metadata/BatchOpsMetadatReindexerTest.java index d8ec3f798b4..9926a905467 100644 --- a/services/src/test/java/org/fao/geonet/services/metadata/BatchOpsMetadatReindexerTest.java +++ b/services/src/test/java/org/fao/geonet/services/metadata/BatchOpsMetadatReindexerTest.java @@ -5,6 +5,7 @@ import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.ISearchManager; import org.fao.geonet.kernel.search.index.BatchOpsMetadataReindexer; +import org.fao.geonet.kernel.search.submission.IIndexSubmitter; import org.fao.geonet.util.ThreadUtils; import org.junit.Test; import org.junit.runner.RunWith; @@ -192,7 +193,7 @@ public Void answer(InvocationOnMock invocationOnMock) throws Throwable { usedTread.add(Thread.currentThread()); return null; } - }).when(mockDataMan).indexMetadata(Mockito.anyString(), Mockito.anyBoolean()); + }).when(mockDataMan).indexMetadata(Mockito.anyString(), Mockito.any(IIndexSubmitter.class)); return mockDataMan; } @@ -206,7 +207,7 @@ public Void answer(InvocationOnMock invocationOnMock) throws Throwable { latch.await(); return null; } - }).when(mockDataMan).indexMetadata(Mockito.anyString(), Mockito.anyBoolean()); + }).when(mockDataMan).indexMetadata(Mockito.anyString(), Mockito.any(IIndexSubmitter.class)); return mockDataMan; } @@ -221,10 +222,10 @@ private Set createMetadataToIndex() { private ArgumentCaptor captureIndexationLaunched(DataManager mockDataMan) throws Exception { ArgumentCaptor metadataIdCaptor = ArgumentCaptor.forClass(String.class); - ArgumentCaptor forceRefreshCaptor = ArgumentCaptor.forClass(Boolean.class); + ArgumentCaptor indexSubmittorCaptor = ArgumentCaptor.forClass(IIndexSubmitter.class); ArgumentCaptor isearchManagerCaptor = ArgumentCaptor.forClass(ISearchManager.class); - Mockito.verify(mockDataMan, Mockito.times(4)).indexMetadata(metadataIdCaptor.capture(), forceRefreshCaptor.capture()); + Mockito.verify(mockDataMan, Mockito.times(4)).indexMetadata(metadataIdCaptor.capture(), indexSubmittorCaptor.capture()); return metadataIdCaptor; } } diff --git a/web-ui/src/main/resources/catalog/js/admin/AdminToolsController.js b/web-ui/src/main/resources/catalog/js/admin/AdminToolsController.js index 393fc0268f1..d8d068bfc93 100644 --- a/web-ui/src/main/resources/catalog/js/admin/AdminToolsController.js +++ b/web-ui/src/main/resources/catalog/js/admin/AdminToolsController.js @@ -252,10 +252,7 @@ $scope.rebuildIndex = function (dropFirst, index) { var url = - "../api/site/index?reset=" + - dropFirst + - "&asynchronous=true" + - (index ? "&indices=" + index : ""); + "../api/site/index?reset=" + dropFirst + (index ? "&indices=" + index : ""); return $http.put(url).then( function (response) { checkIsIndexing(); @@ -271,22 +268,6 @@ ); }; - $scope.commitIndexChanges = function (dropFirst) { - return $http.put("../api/site/index/commit").then( - function (response) { - checkIsIndexing(); - }, - function (response) { - $rootScope.$broadcast("StatusUpdated", { - title: $translate.instant("indexCommitError"), - error: response.data, - timeout: 0, - type: "danger" - }); - } - ); - }; - $scope.clearJsCache = function () { return $http.get("../../static/wroAPI/reloadModel").then(function (response) { $http.get("../../static/wroAPI/reloadCache").then(function (response) { diff --git a/web-ui/src/main/resources/catalog/locales/az-v4.json b/web-ui/src/main/resources/catalog/locales/az-v4.json index 959f7e280ef..c0b8bab5694 100644 --- a/web-ui/src/main/resources/catalog/locales/az-v4.json +++ b/web-ui/src/main/resources/catalog/locales/az-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "İndeksi yenidən qurarkən, axtarış natamam nəticələr verə bilər və CSW GetRecords əməliyyatı deaktiv edilə bilər (əgər parametrlərdə seçimi seçmisinizsə). Kataloq trafiki az olduqda bu funksiyadan istifadə edin. Verilənlər bazasında birbaşa dəyişikliklər edərkən indeksi buradan əl ilə yenidən qurmaq tövsiyə olunur. İndeks xəritələşdirilməsini dəyişdirsəniz (müq. records.json ), onda siz "İndeksi və yenidən indeksi sil" üzərinə klikləməlisiniz.", "indexInEsDoneError": "İndeksdə xəta var. Təfərrüatlar üçün qeydlərə baxın", "indexInEsDone": "İndeksləmə əməliyyatı uğurla keçdi", - "indexCommit": "İndeks dəyişikliklərini qəbul edin", - "indexCommit-help": "Yalnız indeksləşdirmə tapşırığı asılı olduqda istifadə etmək üçün.", - "indexCommitError": "İndeks dəyişiklikləri edərkən xəta baş verdi.", "ui-moreLikeThisConfig": "Bu konfiqurasiyaya daha çox bənzəyir", "ui-moreLikeThisConfig-help": "Konfiqurasiyada oxşar qeydləri axtarmaq üçün qeyd başlığı ilə təyin olunacaq more_like_this.like olmalıdır.", "ui-autocompleteConfig": "Avtomatik tamamlama konfiqurasiyası", diff --git a/web-ui/src/main/resources/catalog/locales/ca-v4.json b/web-ui/src/main/resources/catalog/locales/ca-v4.json index e54c2ab9c57..a90f18808b8 100644 --- a/web-ui/src/main/resources/catalog/locales/ca-v4.json +++ b/web-ui/src/main/resources/catalog/locales/ca-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successfull", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records.", "ui-autocompleteConfig": "Autocompletion configuration", diff --git a/web-ui/src/main/resources/catalog/locales/cs-v4.json b/web-ui/src/main/resources/catalog/locales/cs-v4.json index 46e448ca625..a5982ee8f3d 100644 --- a/web-ui/src/main/resources/catalog/locales/cs-v4.json +++ b/web-ui/src/main/resources/catalog/locales/cs-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "Při obnovování indexu může vyhledávání vracet neúplné výsledky a operace CSR GetRecords může být deaktivována (pokud jste v nastavení vybrali tuto možnost). Tuto funkci použijte, pokud je provoz katalogu nízký. Při provádění změn v databázi se doporučuje odtud znovu vytvořit index ručně. Pokud změníte mapování indexů (srov. records.json), musíte kliknout na 'Smazat index a reindexovat'.", "indexInEsDoneError": "S indexem došlo k chybě. Podrobnosti najdete v protokolech", "indexInEsDone": "Operace indexování byla úspěšná", - "indexCommit": "Potvrdit změny indexu", - "indexCommit-help": "Použít pouze v případě, že úloha indexování visí.", - "indexCommitError": "Chyba při provádění změn indexu.", "ui-moreLikeThisConfig": "Spíše jako tato konfigurace", "ui-moreLikeThisConfig-help": "Konfigurace musí mít more_like_this.like, které budou nastaveny s názvem záznamu pro vyhledávání podobných záznamů.", "ui-autocompleteConfig": "Konfigurace automatického doplňování", diff --git a/web-ui/src/main/resources/catalog/locales/cy-v4.json b/web-ui/src/main/resources/catalog/locales/cy-v4.json index e412c79f700..4cbb0538abe 100644 --- a/web-ui/src/main/resources/catalog/locales/cy-v4.json +++ b/web-ui/src/main/resources/catalog/locales/cy-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successfull", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records.", "ui-autocompleteConfig": "Autocompletion configuration", diff --git a/web-ui/src/main/resources/catalog/locales/da-v4.json b/web-ui/src/main/resources/catalog/locales/da-v4.json index 079d11dee2e..2c4ddb2bec6 100644 --- a/web-ui/src/main/resources/catalog/locales/da-v4.json +++ b/web-ui/src/main/resources/catalog/locales/da-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "Mens indekset genopbygges, kan søgningen returnere ufuldstændige resultater, og CSW GetRecords-operationen kan deaktiveres (hvis du valgte denne mulighed i indstillingerne). Benyt denne funktion, når katalogtrafikken er lav. Det anbefales at genopbygge indekset manuelt herfra, når du foretager ændringer direkte i databasen. Hvis du ændrer indekskortlægning (jf . records.json ), så skal du klikke på 'Slet indeks og genindekser'.", "indexInEsDoneError": "Der er eb fejl i indekset. Se loggen for yderligere detaljer", "indexInEsDone": "Indekserings-operationen var succesfuld ", - "indexCommit": "Foretag ændringer i indekset", - "indexCommit-help": "Anvendes kun, hvis indeksering er kørt fast.", - "indexCommitError": "Fejl ved anvendelse af ændringer i indeks", "ui-moreLikeThisConfig": "Mere som denne konfiguration", "ui-moreLikeThisConfig-help": "Konfigurationen skal have tagget 'more_like_this.like', som bliver sat sammen med instansens titel, for at søge efter lignende instanser.", "ui-autocompleteConfig": "Konfiguration af autocomplete", diff --git a/web-ui/src/main/resources/catalog/locales/de-v4.json b/web-ui/src/main/resources/catalog/locales/de-v4.json index 163db0f6074..ef0682f3417 100644 --- a/web-ui/src/main/resources/catalog/locales/de-v4.json +++ b/web-ui/src/main/resources/catalog/locales/de-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "Während der Neuerstellung des Index gibt die Suche möglicherweise unvollständige Ergebnisse zurück und CSW GetRecords kann deaktiviert werden (wenn dies in den Einstellungen ausgewählt wird). Verwenden Sie die Funktion, wenn der Katalogverkehr gering ist. Es wird empfohlen, den Index von hier aus manuell neu zu erstellen, wenn Sie Änderungen direkt in der Datenbank vornehmen. Wenn Sie die Indexzuordnung ändern (vgl. records.json), müssen Sie auf 'Index löschen und neu indexieren' klicken.", "indexInEsDoneError": "Es liegt ein Fehler im Index vor. Einzelheiten finden Sie in den Logs.", "indexInEsDone": "Die Indexbildung war erfolgreich", - "indexCommit": "Commit Index-Änderungen", - "indexCommit-help": "Nur verwenden, wenn die Indexbildung hängt.", - "indexCommitError": "Fehler beim Schreiben von Indexänderungen.", "ui-moreLikeThisConfig": "Besser / ähnliche wie diese Konfiguration", "ui-moreLikeThisConfig-help": "Die Konfiguration muss ein more_like_this.like haben, das mit dem Datensatztitel festgelegt wird, um nach ähnlichen Datensätzen zu suchen.", "ui-autocompleteConfig": "Konfiguration Auto-Vervollständigen", diff --git a/web-ui/src/main/resources/catalog/locales/en-v4.json b/web-ui/src/main/resources/catalog/locales/en-v4.json index 4876d13015b..665714b02bf 100644 --- a/web-ui/src/main/resources/catalog/locales/en-v4.json +++ b/web-ui/src/main/resources/catalog/locales/en-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successful", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records. (See Elasticsearch API).", "ui-moreLikeThisFilter": "More like this query filter", diff --git a/web-ui/src/main/resources/catalog/locales/es-v4.json b/web-ui/src/main/resources/catalog/locales/es-v4.json index 7e4aea85a23..9102a138e3e 100644 --- a/web-ui/src/main/resources/catalog/locales/es-v4.json +++ b/web-ui/src/main/resources/catalog/locales/es-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "Cuando se está reconstruyendo el índice la búsqueda podría devolver resultados incompletos y la operación GetRecords puede mostrarse como deshabilitada (si se ha seleccionado la opción en la configuración). Usa esta función cuando el tráfico del catálogo sea bajo. Se recomienda reconstruir el índice desde aquí cuando se hayan hecho cambios directamente en la base de datos. Si se cambia la configuración del índice (fichero records.json) hay que usar 'Borrar índice y reindexar'.", "indexInEsDoneError": "Hay un error con el índice. Consulte los logs para obtener más detalles", "indexInEsDone": "La operación de indexación fue exitosa", - "indexCommit": "Confirmar los cambios del índice", - "indexCommit-help": "Usar solo si la tarea de indexación está bloqueada.", - "indexCommitError": "Error al realizar cambios en el índice.", "ui-moreLikeThisConfig": "Configuración de \"Más como éste\"", "ui-moreLikeThisConfig-help": "La configuración debe tener un more_like_this.like que se establecerá con el título del registro para buscar registros similares.", "ui-autocompleteConfig": "Configuración del autocompletado", diff --git a/web-ui/src/main/resources/catalog/locales/fi-v4.json b/web-ui/src/main/resources/catalog/locales/fi-v4.json index 7c9a0264cf0..c1688a98e8c 100644 --- a/web-ui/src/main/resources/catalog/locales/fi-v4.json +++ b/web-ui/src/main/resources/catalog/locales/fi-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successfull", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records.", "ui-autocompleteConfig": "Autocompletion configuration", diff --git a/web-ui/src/main/resources/catalog/locales/fr-v4.json b/web-ui/src/main/resources/catalog/locales/fr-v4.json index d943adf7563..7976b0a39c7 100644 --- a/web-ui/src/main/resources/catalog/locales/fr-v4.json +++ b/web-ui/src/main/resources/catalog/locales/fr-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "Lors de la reconstruction de l'index, la recherche peut retourner des résultats incomplets et l'opération GetRecords du CSW peut être désactivée (selon votre configuration). Utiliser cette fonction lorsque que le traffic au niveau du catalogue est faible. Il est recommandé de reconstruire l'index après avoir effectuer des changements directement dans la base de données. Si vous modifier le schéma de l'index (cf. records.json), alors vous devez supprimer et recréer l'index.", "indexInEsDoneError": "Il y a un problème avec l'index. Vérifier les logs pour plus d'information.", "indexInEsDone": "L'indexation s'est bien déroulée.", - "indexCommit": "Committer les changements dans l'index", - "indexCommit-help": "Utiliser cette action uniquement si l'indexation reste en attente.", - "indexCommitError": "Erreur lors du commit", "ui-moreLikeThisConfig": "Configuration pour le calcul de similarité entre fiches", "ui-moreLikeThisConfig-help": "La configuration DOIT avoir un élément more_like_this.like qui sera défini avec la valeur du titre de la fiche en cours de consultation.", "ui-autocompleteConfig": "Configuration pour l'auto-completion", diff --git a/web-ui/src/main/resources/catalog/locales/hy-v4.json b/web-ui/src/main/resources/catalog/locales/hy-v4.json index 1a4572b8032..4c18b2f05f1 100644 --- a/web-ui/src/main/resources/catalog/locales/hy-v4.json +++ b/web-ui/src/main/resources/catalog/locales/hy-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "Ինդեքսը վերակառուցելիս որոնումը կարող է վերադարձնել թերի արդյունքներ, և CSW GetRecords-ի գործողությունը կարող է անջատվել (եթե ընտրել եք այդ տարբերակը կարգավորումներում): Օգտագործեք այս գործառույթը, երբ կատալոգի թրաֆիկը քիչ է: Խորհուրդ է տրվում ինդեքսը ձեռքով վերակառուցել այստեղից՝ անմիջապես տվյալների բազայում փոփոխություններ կատարելիս: Եթե փոխում եք ինդեքսների քարտեզագրումը (տես records.json ), ապա պետք է սեղմեք «Ջնջել ինդեքսը և վերաինդեքսը»:", "indexInEsDoneError": "Ցուցանիշի հետ կապված սխալ կա: Մանրամասների համար տես տեղեկամատյանները", "indexInEsDone": "Ինդեքսավորման գործողությունը հաջողությամբ ավարտվեց", - "indexCommit": "Կատարել ինդեքսի փոփոխությունները", - "indexCommit-help": "Օգտագործել միայն այն դեպքում, եթե ինդեքսավորման առաջադրանքը կախված է:", - "indexCommitError": "Սխալ՝ ինդեքսի փոփոխություններ կատարելիս:", "ui-moreLikeThisConfig": "Ավելի շատ նման է այս կազմաձևմանը", "ui-moreLikeThisConfig-help": "Կազմաձևը պետք է ունենա more_like_this.like, որը կսահմանվի ռեկորդի վերնագրի հետ՝ նմանատիպ գրառումներ որոնելու համար:", "ui-autocompleteConfig": "Ավտոմատ լրացման կարգավորում", diff --git a/web-ui/src/main/resources/catalog/locales/is-v4.json b/web-ui/src/main/resources/catalog/locales/is-v4.json index c3297cfa1ab..67ccd6fa7f0 100644 --- a/web-ui/src/main/resources/catalog/locales/is-v4.json +++ b/web-ui/src/main/resources/catalog/locales/is-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successfull", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records.", "ui-autocompleteConfig": "Autocompletion configuration", diff --git a/web-ui/src/main/resources/catalog/locales/it-v4.json b/web-ui/src/main/resources/catalog/locales/it-v4.json index d5ccebd01fe..3fa974fd831 100644 --- a/web-ui/src/main/resources/catalog/locales/it-v4.json +++ b/web-ui/src/main/resources/catalog/locales/it-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successfull", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records.", "ui-autocompleteConfig": "Autocompletion configuration", diff --git a/web-ui/src/main/resources/catalog/locales/ka-v4.json b/web-ui/src/main/resources/catalog/locales/ka-v4.json index 87fdebf6ed7..a8206e848c4 100644 --- a/web-ui/src/main/resources/catalog/locales/ka-v4.json +++ b/web-ui/src/main/resources/catalog/locales/ka-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "ინდექსის აღდგენისას, ძიებამ შეიძლება დააბრუნოს არასრული შედეგები და CSW GetRecords ოპერაცია შეიძლება გამორთოთ (თუ პარამეტრებში აირჩიეთ ვარიანტი). გამოიყენეთ ეს ფუნქცია, როდესაც კატალოგის ტრაფიკი დაბალია. რეკომენდირებულია აქედან ხელით აღადგინოთ ინდექსი, როდესაც უშუალოდ მონაცემთა ბაზაში შეიტანეთ ცვლილებები. თუ თქვენ შეცვლით ინდექსის რუკს (შდრ. records.json ), მაშინ უნდა დააჭიროთ 'Delete index and reindex'.", "indexInEsDoneError": "ინდექსთან არის შეცდომა. იხილეთ ჟურნალები დეტალებისთვის", "indexInEsDone": "ინდექსირების ოპერაცია წარმატებით დასრულდა", - "indexCommit": "ინდექსის ცვლილებების შესრულება", - "indexCommit-help": "გამოყენება მხოლოდ იმ შემთხვევაში, თუ ინდექსირების ამოცანა ჩამოკიდებულია.", - "indexCommitError": "შეცდომა ინდექსის ცვლილებების შესრულებისას.", "ui-moreLikeThisConfig": "უფრო მეტად მოსწონს ეს კონფიგურაცია", "ui-moreLikeThisConfig-help": "კონფიგურაციას უნდა ჰქონდეს more_like_this.like, რომელიც დაყენებული იქნება ჩანაწერის სათაურით მსგავსი ჩანაწერების მოსაძებნად.", "ui-autocompleteConfig": "ავტომატური დასრულების კონფიგურაცია", diff --git a/web-ui/src/main/resources/catalog/locales/ko-v4.json b/web-ui/src/main/resources/catalog/locales/ko-v4.json index 454d968b0f7..e784bffbde4 100644 --- a/web-ui/src/main/resources/catalog/locales/ko-v4.json +++ b/web-ui/src/main/resources/catalog/locales/ko-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successfull", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records.", "ui-autocompleteConfig": "Autocompletion configuration", diff --git a/web-ui/src/main/resources/catalog/locales/nl-v4.json b/web-ui/src/main/resources/catalog/locales/nl-v4.json index 1e56731a75a..253dd7d6cb4 100644 --- a/web-ui/src/main/resources/catalog/locales/nl-v4.json +++ b/web-ui/src/main/resources/catalog/locales/nl-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "Tijdens het opnieuw opbouwen van de index kan het zoeken onvolledige resultaten opleveren en kan de bewerking CSW GetRecords worden uitgeschakeld (als u de optie in de instellingen hebt geselecteerd). Gebruik deze functie als er weinig catalogusverkeer is. Het wordt aanbevolen om de index handmatig opnieuw op te bouwen vanaf hier wanneer u wijzigingen rechtstreeks in de database aanbrengt. Als je de indextoewijzing wijzigt (cf. records.json ), dan moet je klikken op 'Index verwijderen en opnieuw indexeren'.", "indexInEsDoneError": "Er is een fout inde index. Bekijk de logs voor meer details.", "indexInEsDone": "Het indexeren is gelukt", - "indexCommit": "Wijzigingen in de index bevestigen", - "indexCommit-help": "Alleen gebruiken indien de indexatie taak vastgelopen is.", - "indexCommitError": "Er is een fout opgetreden bij het bevestigen van de index wijzigingen", "ui-moreLikeThisConfig": "Configuratie voor vergelijkbare resultaten", "ui-moreLikeThisConfig-help": "De configuratie moet een more_like_this.like hebben die zal worden ingesteld met de recordtitel om te zoeken naar vergelijkbare records.", "ui-autocompleteConfig": "Auto aanvullen configuratie", diff --git a/web-ui/src/main/resources/catalog/locales/pt-v4.json b/web-ui/src/main/resources/catalog/locales/pt-v4.json index 0cebd4cfc5d..282fa8cfbc5 100644 --- a/web-ui/src/main/resources/catalog/locales/pt-v4.json +++ b/web-ui/src/main/resources/catalog/locales/pt-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successfull", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records.", "ui-autocompleteConfig": "Autocompletion configuration", diff --git a/web-ui/src/main/resources/catalog/locales/ro-v4.json b/web-ui/src/main/resources/catalog/locales/ro-v4.json index cc5bf3a7857..5c5bdda18c1 100644 --- a/web-ui/src/main/resources/catalog/locales/ro-v4.json +++ b/web-ui/src/main/resources/catalog/locales/ro-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "În timpul reconstrucției indexului, căutarea poate returna rezultate incomplete și operațiunea CSW GetRecords poate fi dezactivată (dacă ați selectat opțiunea din setări). Utilizați această funcție când traficul de catalog este scăzut. Este recomandat să reconstruiți manual indexul de aici atunci când faceți modificări direct în baza de date. Dacă modificați maparea indexului (cf. records.json ), atunci trebuie să faceți clic pe „Ștergeți indexul și reindexați”.", "indexInEsDoneError": "A apărut o eroare cu indexul. Consultați jurnalele pentru detalii", "indexInEsDone": "Operația de indexare a avut succes", - "indexCommit": "Schimbări ale indexului", - "indexCommit-help": "A se utiliza numai dacă sarcina de indexare este suspendată.", - "indexCommitError": "Eroare la efectuarea modificărilor indexului.", "ui-moreLikeThisConfig": "Mai mult ca această configurație", "ui-moreLikeThisConfig-help": "Configurația trebuie să aibă un more_like_this.like care va fi setat cu titlul înregistrării pentru a căuta înregistrări similare.", "ui-autocompleteConfig": "Configurație de completare automată", diff --git a/web-ui/src/main/resources/catalog/locales/ru-v4.json b/web-ui/src/main/resources/catalog/locales/ru-v4.json index 40f202e1ed4..12b381344d0 100644 --- a/web-ui/src/main/resources/catalog/locales/ru-v4.json +++ b/web-ui/src/main/resources/catalog/locales/ru-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successfull", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records.", "ui-autocompleteConfig": "Autocompletion configuration", diff --git a/web-ui/src/main/resources/catalog/locales/sk-v4.json b/web-ui/src/main/resources/catalog/locales/sk-v4.json index 6348aa506a7..99f078598c5 100644 --- a/web-ui/src/main/resources/catalog/locales/sk-v4.json +++ b/web-ui/src/main/resources/catalog/locales/sk-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successfull", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records.", "ui-autocompleteConfig": "Autocompletion configuration", diff --git a/web-ui/src/main/resources/catalog/locales/sv-v4.json b/web-ui/src/main/resources/catalog/locales/sv-v4.json index 6182b700118..bb86e19b01d 100644 --- a/web-ui/src/main/resources/catalog/locales/sv-v4.json +++ b/web-ui/src/main/resources/catalog/locales/sv-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "Medan index byggs om kan sökningen returnera ofullständiga resultat och CSW GetRecords-funktionen kan inaktiveras (om du valde det alternativet i inställningarna). Använd den här funktionen när katalogtrafiken är låg. Det rekommenderas att bygga om index manuellt härifrån om du gör ändringar direkt i databasen. Om du ändrar indexmappning måste du klicka på 'Ta bort index och indexera om'.", "indexInEsDoneError": "Det finns ett fel med indexet. Se loggarna för mer information", "indexInEsDone": "Indexeringen lyckades", - "indexCommit": "Spara indexändringar", - "indexCommit-help": "Används endast om indexeringen hakat upp sig.", - "indexCommitError": "Fel vid indexändringar.", "ui-moreLikeThisConfig": "Mer som den här konfigurationen", "ui-moreLikeThisConfig-help": "Konfigurationen måste ha en more_like_this.like som kommer att ställas in med posttiteln för att söka efter liknande poster.", "ui-autocompleteConfig": "Konfiguration av autokomplettering", diff --git a/web-ui/src/main/resources/catalog/locales/uk-v4.json b/web-ui/src/main/resources/catalog/locales/uk-v4.json index 0d75716270a..4b99c230161 100644 --- a/web-ui/src/main/resources/catalog/locales/uk-v4.json +++ b/web-ui/src/main/resources/catalog/locales/uk-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "Під час перебудови індексу пошук може видати неповні результати, а операцію CSW GetRecords можна вимкнути (якщо ви вибрали відповідний параметр у налаштуваннях). Використовуйте цю функцію, коли трафік каталогу низький. Рекомендується перебудувати індекс вручну звідси, якщо вносити зміни безпосередньо в базу даних. Якщо ви змінюєте відображення індексу (див. records.json ), то вам потрібно натиснути «Видалити індекс і повторний індекс».", "indexInEsDoneError": "Виникла помилка з індексом. Подробиці дивіться в журналах", "indexInEsDone": "Операція індексації пройшла успішно", - "indexCommit": "Зафіксувати зміни індексу", - "indexCommit-help": "Використовується, лише якщо завдання індексування зависає.", - "indexCommitError": "Помилка під час фіксації змін індексу.", "ui-moreLikeThisConfig": "Більше подібної конфігурації", "ui-moreLikeThisConfig-help": "Конфігурація повинна мати more_like_this.like, який буде встановлено з назвою запису для пошуку подібних записів.", "ui-autocompleteConfig": "Конфігурація автозавершення", diff --git a/web-ui/src/main/resources/catalog/locales/zh-v4.json b/web-ui/src/main/resources/catalog/locales/zh-v4.json index 016f987cb77..1aaea358618 100644 --- a/web-ui/src/main/resources/catalog/locales/zh-v4.json +++ b/web-ui/src/main/resources/catalog/locales/zh-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successfull", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records.", "ui-autocompleteConfig": "Autocompletion configuration", diff --git a/web-ui/src/main/resources/catalog/templates/admin/tools/index.html b/web-ui/src/main/resources/catalog/templates/admin/tools/index.html index adc51c0c483..1c863a7ca7f 100644 --- a/web-ui/src/main/resources/catalog/templates/admin/tools/index.html +++ b/web-ui/src/main/resources/catalog/templates/admin/tools/index.html @@ -39,22 +39,6 @@

indexingInProgress

rebuildIndexHelp
-
-
-
- -
-
indexCommit-help
-
-
diff --git a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsWFSFeatureIndexer.java b/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsWFSFeatureIndexer.java index 3d0857ddbb6..0d50c34dc44 100644 --- a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsWFSFeatureIndexer.java +++ b/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsWFSFeatureIndexer.java @@ -26,14 +26,7 @@ import co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester; import co.elastic.clients.elasticsearch._helpers.bulk.BulkListener; import co.elastic.clients.elasticsearch._types.Result; -import co.elastic.clients.elasticsearch.core.BulkRequest; -import co.elastic.clients.elasticsearch.core.BulkResponse; -import co.elastic.clients.elasticsearch.core.IndexRequest; -import co.elastic.clients.elasticsearch.core.IndexResponse; -import co.elastic.clients.elasticsearch.core.bulk.BulkOperation; -import co.elastic.clients.elasticsearch.core.bulk.IndexOperation; -import co.elastic.clients.json.JsonData; -import co.elastic.clients.json.JsonpMapper; +import co.elastic.clients.elasticsearch.core.*; import co.elastic.clients.util.BinaryData; import co.elastic.clients.util.ContentType; import com.fasterxml.jackson.core.JsonProcessingException; @@ -43,7 +36,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; -import jakarta.json.spi.JsonProvider; import org.apache.camel.Exchange; import org.apache.camel.spring.SpringCamelContext; import org.apache.jcs.access.exception.InvalidArgumentException; @@ -74,7 +66,6 @@ import org.springframework.beans.factory.annotation.Value; import java.io.IOException; -import java.io.StringReader; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; @@ -84,9 +75,6 @@ import java.time.temporal.ChronoUnit; import java.util.*; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Phaser; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; @@ -233,11 +221,11 @@ public void deleteFeatures(String url, String typeName, EsRestClient client) { new Object[]{url, typeName, index, indexType}); try { long begin = System.currentTimeMillis(); - client.deleteByQuery(index, String.format("+featureTypeId:\"%s\"", getIdentifier(url, typeName))); + deleteByQuery(String.format("+featureTypeId:\"%s\"", getIdentifier(url, typeName))); LOGGER.info(" Features deleted in {} ms.", System.currentTimeMillis() - begin); begin = System.currentTimeMillis(); - client.deleteByQuery(index, String.format("+id:\"%s\"", + deleteByQuery(String.format("+id:\"%s\"", getIdentifier(url, typeName))); LOGGER.info(" Report deleted in {} ms.", System.currentTimeMillis() - begin); @@ -247,6 +235,21 @@ public void deleteFeatures(String url, String typeName, EsRestClient client) { } } + private void deleteByQuery(String query) throws IOException { + DeleteByQueryRequest deleteByQueryRequest = client.buildDeleteByQuery(index, query); + DeleteByQueryResponse deleteByQueryResponse = client.getClient().deleteByQuery(deleteByQueryRequest); + if (!deleteByQueryResponse.failures().isEmpty()) { + StringBuilder stringBuilder = new StringBuilder(); + + deleteByQueryResponse.failures().forEach(f -> stringBuilder.append(f.toString())); + + throw new RuntimeException(String.format( + "Error during removal of query %s. Errors are '%s'.", query, stringBuilder.toString() + )); + } + + } + interface TitleResolver { void setTitle(ObjectNode objectNode, SimpleFeature simpleFeature); } @@ -634,7 +637,7 @@ public void afterBulk(long executionId, BulkRequest request, List contex } }; - this.bulk = BulkIngester.of(b -> b.client(client.getAsynchClient()) + this.bulk = BulkIngester.of(b -> b.client(client.getAsyncClient()) .listener(listener) // .maxConcurrentRequests(1) // .flushInterval(10, TimeUnit.SECONDS)