From 9655e7e273ef04e5d8def895bdbbb09ed42a3228 Mon Sep 17 00:00:00 2001 From: mingan <11653241+mingan666@users.noreply.github.com> Date: Thu, 29 Dec 2022 15:58:52 +0100 Subject: [PATCH 1/5] Fix cmd line version --- .../src/main/java/org/piwigo/remotesync/api/Main.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/remotesync-api/src/main/java/org/piwigo/remotesync/api/Main.java b/remotesync-api/src/main/java/org/piwigo/remotesync/api/Main.java index db0265d..756fb9b 100644 --- a/remotesync-api/src/main/java/org/piwigo/remotesync/api/Main.java +++ b/remotesync-api/src/main/java/org/piwigo/remotesync/api/Main.java @@ -4,13 +4,15 @@ * are made available under the terms of the GNU Public License v2.0 * which accompanies this distribution, and is available at * http://www.gnu.org/licenses/old-licenses/gpl-2.0.html - * + * * Contributors: * Matthieu Helleboid - initial API and implementation ******************************************************************************/ package org.piwigo.remotesync.api; import org.piwigo.remotesync.api.AbstractMain; +import org.piwigo.remotesync.api.sync.LoginJob; +import org.piwigo.remotesync.api.sync.SyncJob; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,9 +22,12 @@ public class Main extends AbstractMain { public static void main(String[] args) { new Main().run(args); } - + protected void start() { logger.debug("will start batch Remotesync"); + LoginJob preJob = new LoginJob(); + preJob.execute(); + new SyncJob(preJob).execute(); } // // TODO implement dry run From c7017a3146fc41f7861f1a321697b1e019fd97f2 Mon Sep 17 00:00:00 2001 From: mingan <11653241+mingan666@users.noreply.github.com> Date: Thu, 29 Dec 2022 17:03:50 +0100 Subject: [PATCH 2/5] Store table in database for redirect service --- .gitignore | 1 + remotesync-api/.gitignore | 1 + remotesync-api/pom.xml | 24 +++++- .../api/sync/SyncDirectoryWalker.java | 13 +++- .../piwigo/remotesync/menalto/ImportItem.java | 75 +++++++++++++++++++ .../piwigo/remotesync/menalto/Importer.java | 39 ++++++++++ .../menalto/PersistenceManager.java | 48 ++++++++++++ .../META-INF/persistence.xml.example | 15 ++++ 8 files changed, 212 insertions(+), 4 deletions(-) create mode 100644 .gitignore create mode 100644 remotesync-api/src/main/java/org/piwigo/remotesync/menalto/ImportItem.java create mode 100644 remotesync-api/src/main/java/org/piwigo/remotesync/menalto/Importer.java create mode 100644 remotesync-api/src/main/java/org/piwigo/remotesync/menalto/PersistenceManager.java create mode 100644 remotesync-api/src/main/resources/META-INF/persistence.xml.example diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..485dee6 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea diff --git a/remotesync-api/.gitignore b/remotesync-api/.gitignore index b83d222..4f54c4e 100644 --- a/remotesync-api/.gitignore +++ b/remotesync-api/.gitignore @@ -1 +1,2 @@ /target/ +persistence.xml diff --git a/remotesync-api/pom.xml b/remotesync-api/pom.xml index e6204c9..2aa53d0 100644 --- a/remotesync-api/pom.xml +++ b/remotesync-api/pom.xml @@ -4,7 +4,7 @@ are made available under the terms of the GNU Public License v2.0 which accompanies this distribution, and is available at http://www.gnu.org/licenses/old-licenses/gpl-2.0.html - + Contributors: Matthieu Helleboid - initial API and implementation --> @@ -13,7 +13,29 @@ 4.0.0 remotesync-api Piwigo Remote Sync API + + + 5.6.14.Final + 3.0.6 + + + + org.hibernate + hibernate-core-jakarta + ${hibernate.version} + + + org.mariadb.jdbc + mariadb-java-client + ${mariadb.version} + + + org.apache.commons + commons-lang3 + 3.12.0 + + args4j args4j diff --git a/remotesync-api/src/main/java/org/piwigo/remotesync/api/sync/SyncDirectoryWalker.java b/remotesync-api/src/main/java/org/piwigo/remotesync/api/sync/SyncDirectoryWalker.java index 7635d13..aa94e39 100644 --- a/remotesync-api/src/main/java/org/piwigo/remotesync/api/sync/SyncDirectoryWalker.java +++ b/remotesync-api/src/main/java/org/piwigo/remotesync/api/sync/SyncDirectoryWalker.java @@ -4,7 +4,7 @@ * are made available under the terms of the GNU Public License v2.0 * which accompanies this distribution, and is available at * http://www.gnu.org/licenses/old-licenses/gpl-2.0.html - * + * * Contributors: * Matthieu Helleboid - initial API and implementation ******************************************************************************/ @@ -20,6 +20,7 @@ import org.piwigo.remotesync.api.Constants; import org.piwigo.remotesync.api.ISyncConfiguration; import org.piwigo.remotesync.api.cache.LegacyCache; +import org.piwigo.remotesync.menalto.Importer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,10 +31,12 @@ public abstract class SyncDirectoryWalker extends DirectoryWalker { protected ISyncConfiguration syncConfiguration; protected File startDirectory; protected Map legacyCaches = new HashMap(); + private Importer importer; protected SyncDirectoryWalker(ISyncConfiguration syncConfiguration) { super(null, Constants.IMAGE_EXTENSIONS_FILTER, -1); this.syncConfiguration = syncConfiguration; + importer = new Importer(syncConfiguration.getDirectory()); } @Override @@ -55,7 +58,9 @@ protected void handleDirectoryStart(File directory, int depth, Collection // FIXME : ignore me } logger.info("Creating album for " + directory); - legacyCache.addAlbum(createAlbum(directory, parentAlbumId)); + Integer albumId = createAlbum(directory, parentAlbumId); + legacyCache.addAlbum(albumId); + importer.addAlbum(directory.getAbsolutePath(), albumId); } } @@ -72,7 +77,9 @@ protected void handleFile(File file, int depth, java.util.Collection resul // FIXME : ignore me } logger.info("Uploading " + file.getName() + " in album with ID " + albumId); - legacyCache.addImage(file, createImage(file, albumId)); + Integer imageId = createImage(file, albumId); + legacyCache.addImage(file, imageId); + importer.addImage(file.getAbsolutePath(), albumId, imageId); } } diff --git a/remotesync-api/src/main/java/org/piwigo/remotesync/menalto/ImportItem.java b/remotesync-api/src/main/java/org/piwigo/remotesync/menalto/ImportItem.java new file mode 100644 index 0000000..17430a7 --- /dev/null +++ b/remotesync-api/src/main/java/org/piwigo/remotesync/menalto/ImportItem.java @@ -0,0 +1,75 @@ +package org.piwigo.remotesync.menalto; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.NamedQueries; +import jakarta.persistence.NamedQuery; +import jakarta.persistence.Table; + +@Entity +@Table(name = "import_item") +@NamedQueries({ + @NamedQuery(name = "ImportItem.findAll", query = "select t from ImportItem t") +}) +public class ImportItem +{ + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id", nullable = false) + private Integer myId; + + @Column(name = "path", nullable = false, length = 255) + private String myPath; + + @Column(name = "album_id", nullable = false) + private Integer myAlbumId; + + @Column(name = "image_id", nullable = true) + private Integer myImageId; + + public Integer getId() + { + return myId; + } + + public void setId(Integer id) + { + myId = id; + } + + public String getPath() + { + return myPath; + } + + public ImportItem setPath(String path) + { + myPath = path; + return this; + } + + public Integer getAlbumId() + { + return myAlbumId; + } + + public ImportItem setAlbumId(Integer albumId) + { + myAlbumId = albumId; + return this; + } + + public Integer getImageId() + { + return myImageId; + } + + public ImportItem setImageId(Integer imageId) + { + myImageId = imageId; + return this; + } +} diff --git a/remotesync-api/src/main/java/org/piwigo/remotesync/menalto/Importer.java b/remotesync-api/src/main/java/org/piwigo/remotesync/menalto/Importer.java new file mode 100644 index 0000000..bf5d762 --- /dev/null +++ b/remotesync-api/src/main/java/org/piwigo/remotesync/menalto/Importer.java @@ -0,0 +1,39 @@ +package org.piwigo.remotesync.menalto; + +import jakarta.persistence.EntityManager; + +public class Importer +{ + private final EntityManager myEntityManager; + private final String myStartDirectory; + + public Importer(String directory) + { + PersistenceManager persistenceManager = new PersistenceManager(); + myEntityManager = persistenceManager.getEntityManager(); + myStartDirectory = directory; + } + + public void addAlbum(String path, Integer albumId) + { + addImportItem(new ImportItem().setPath(getRelativePath(path)).setAlbumId(albumId)); + } + + public void addImage(String path, Integer albumId, Integer imageId) + { + addImportItem(new ImportItem().setPath(getRelativePath(path)).setAlbumId(albumId).setImageId(imageId)); + } + + private String getRelativePath(String path) + { + String relative = path.replace(myStartDirectory, ""); + return relative = relative.startsWith("/") ? relative.substring(1) : relative; + } + + private void addImportItem(ImportItem importItem) + { + myEntityManager.getTransaction().begin(); + myEntityManager.persist(importItem); + myEntityManager.getTransaction().commit(); + } +} diff --git a/remotesync-api/src/main/java/org/piwigo/remotesync/menalto/PersistenceManager.java b/remotesync-api/src/main/java/org/piwigo/remotesync/menalto/PersistenceManager.java new file mode 100644 index 0000000..9d5e220 --- /dev/null +++ b/remotesync-api/src/main/java/org/piwigo/remotesync/menalto/PersistenceManager.java @@ -0,0 +1,48 @@ +package org.piwigo.remotesync.menalto; + +import jakarta.persistence.EntityManager; +import jakarta.persistence.EntityManagerFactory; +import jakarta.persistence.Persistence; + +public class PersistenceManager +{ + public static final String PERSISTENCE_UNIT_NAME = "gallery3"; + + private final EntityManagerFactory myFactory; + private final EntityManager myEntityManager; + + public PersistenceManager() + { + try + { + myFactory = Persistence.createEntityManagerFactory(PERSISTENCE_UNIT_NAME); + myEntityManager = myFactory.createEntityManager(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + + public void shutdown() + { + if (myEntityManager != null && myEntityManager.isOpen()) + { + myEntityManager.close(); + } + + if (myFactory != null && myFactory.isOpen()) + { + myFactory.close(); + } + } + + public EntityManager getEntityManager() + { + if (myEntityManager != null && myEntityManager.isOpen()) + { + return myEntityManager; + } + throw new RuntimeException("No database connection for persistence unit: " + PERSISTENCE_UNIT_NAME); + } +} diff --git a/remotesync-api/src/main/resources/META-INF/persistence.xml.example b/remotesync-api/src/main/resources/META-INF/persistence.xml.example new file mode 100644 index 0000000..bff625a --- /dev/null +++ b/remotesync-api/src/main/resources/META-INF/persistence.xml.example @@ -0,0 +1,15 @@ + + + + se.rutgers.gallery.persistence.menalto.ImportItem + + + + + + + + + + + From 16e51b0f733b8a4b2ba9d1bf790a7d757f5b1bd8 Mon Sep 17 00:00:00 2001 From: mingan <11653241+mingan666@users.noreply.github.com> Date: Sat, 11 Feb 2023 23:15:13 +0100 Subject: [PATCH 3/5] Add url from menalto db to ImportItem --- .../piwigo/remotesync/menalto/ImportItem.java | 14 ++++ .../piwigo/remotesync/menalto/Importer.java | 41 ++++++++++-- .../org/piwigo/remotesync/menalto/Item.java | 65 +++++++++++++++++++ 3 files changed, 116 insertions(+), 4 deletions(-) create mode 100644 remotesync-api/src/main/java/org/piwigo/remotesync/menalto/Item.java diff --git a/remotesync-api/src/main/java/org/piwigo/remotesync/menalto/ImportItem.java b/remotesync-api/src/main/java/org/piwigo/remotesync/menalto/ImportItem.java index 17430a7..1f9bd4b 100644 --- a/remotesync-api/src/main/java/org/piwigo/remotesync/menalto/ImportItem.java +++ b/remotesync-api/src/main/java/org/piwigo/remotesync/menalto/ImportItem.java @@ -24,6 +24,9 @@ public class ImportItem @Column(name = "path", nullable = false, length = 255) private String myPath; + @Column(name = "url", nullable = false, length = 255) + private String myUrl; + @Column(name = "album_id", nullable = false) private Integer myAlbumId; @@ -51,6 +54,17 @@ public ImportItem setPath(String path) return this; } + public String getUrl() + { + return myUrl; + } + + public ImportItem setUrl(String url) + { + myUrl = url; + return this; + } + public Integer getAlbumId() { return myAlbumId; diff --git a/remotesync-api/src/main/java/org/piwigo/remotesync/menalto/Importer.java b/remotesync-api/src/main/java/org/piwigo/remotesync/menalto/Importer.java index bf5d762..8993ead 100644 --- a/remotesync-api/src/main/java/org/piwigo/remotesync/menalto/Importer.java +++ b/remotesync-api/src/main/java/org/piwigo/remotesync/menalto/Importer.java @@ -1,9 +1,15 @@ package org.piwigo.remotesync.menalto; import jakarta.persistence.EntityManager; +import org.piwigo.remotesync.api.sync.SyncDirectoryWalker; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class Importer { + private static final Logger logger = LoggerFactory.getLogger(Importer.class); + + private static final String PREFIX = "albums"; private final EntityManager myEntityManager; private final String myStartDirectory; @@ -11,23 +17,36 @@ public Importer(String directory) { PersistenceManager persistenceManager = new PersistenceManager(); myEntityManager = persistenceManager.getEntityManager(); - myStartDirectory = directory; + myStartDirectory = directory + "/" + PREFIX; + } + + public boolean shouldImport(String path) + { + return path.startsWith(myStartDirectory); } public void addAlbum(String path, Integer albumId) { - addImportItem(new ImportItem().setPath(getRelativePath(path)).setAlbumId(albumId)); + addImportItem(path, albumId, null); } public void addImage(String path, Integer albumId, Integer imageId) { - addImportItem(new ImportItem().setPath(getRelativePath(path)).setAlbumId(albumId).setImageId(imageId)); + addImportItem(path, albumId, imageId); + } + + private void addImportItem(String path, Integer albumId, Integer imageId) + { + String relativePath = getRelativePath(path); + Item item = forPath(relativePath); + ImportItem importItem = new ImportItem().setPath(relativePath).setAlbumId(albumId).setImageId(imageId).setUrl(item.getUrl()); + addImportItem(importItem); } private String getRelativePath(String path) { String relative = path.replace(myStartDirectory, ""); - return relative = relative.startsWith("/") ? relative.substring(1) : relative; + return relative.startsWith("/") ? relative.substring(1) : relative; } private void addImportItem(ImportItem importItem) @@ -36,4 +55,18 @@ private void addImportItem(ImportItem importItem) myEntityManager.persist(importItem); myEntityManager.getTransaction().commit(); } + + private Item forPath(String path) + { + try + { + return myEntityManager.createNamedQuery(Item.FIND_BY_PATH, Item.class) + .setParameter(Item.PATH, path) + .getSingleResult(); + } catch (Exception e) + { + logger.info("Error fetch info for entity with path:" + path); + throw e; + } + } } diff --git a/remotesync-api/src/main/java/org/piwigo/remotesync/menalto/Item.java b/remotesync-api/src/main/java/org/piwigo/remotesync/menalto/Item.java new file mode 100644 index 0000000..e5605c7 --- /dev/null +++ b/remotesync-api/src/main/java/org/piwigo/remotesync/menalto/Item.java @@ -0,0 +1,65 @@ +package org.piwigo.remotesync.menalto; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.NamedQueries; +import jakarta.persistence.NamedQuery; +import jakarta.persistence.Table; + +@Entity +@Table(name = "items") +@NamedQueries({ + @NamedQuery(name = Item.FIND_BY_PATH, query = "select i from Item i where i.myPath =:path"), +}) +public class Item +{ + public static final String FIND_BY_PATH = "Item.findByPath"; + public static final String PATH = "path"; + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id", nullable = false) + private Integer myId; + + @Column(name = "relative_path_cache", nullable = false, length = 255) + private String myPath; + + @Column(name = "relative_url_cache", nullable = false, length = 255) + private String myUrl; + + public Integer getId() + { + return myId; + } + + public Item setId(Integer id) + { + myId = id; + return this; + } + + public String getPath() + { + return myPath; + } + + public Item setPath(String path) + { + myPath = path; + return this; + } + + public String getUrl() + { + return myUrl; + } + + public Item setUrl(String url) + { + myUrl = url; + return this; + } +} From 5c1376f3faa96e15a0d934e6b1a1164f95d08a32 Mon Sep 17 00:00:00 2001 From: mingan <11653241+mingan666@users.noreply.github.com> Date: Sat, 11 Feb 2023 23:17:32 +0100 Subject: [PATCH 4/5] Add menalto albums directory as expected top album --- .../org/piwigo/remotesync/api/sync/SyncDirectoryWalker.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/remotesync-api/src/main/java/org/piwigo/remotesync/api/sync/SyncDirectoryWalker.java b/remotesync-api/src/main/java/org/piwigo/remotesync/api/sync/SyncDirectoryWalker.java index aa94e39..19dec46 100644 --- a/remotesync-api/src/main/java/org/piwigo/remotesync/api/sync/SyncDirectoryWalker.java +++ b/remotesync-api/src/main/java/org/piwigo/remotesync/api/sync/SyncDirectoryWalker.java @@ -48,6 +48,9 @@ protected void handleDirectoryStart(File directory, int depth, Collection if (directory.equals(startDirectory)) return; + if( !importer.shouldImport(directory.getAbsolutePath())) + return; + if (legacyCache.getAlbumCacheElement() != null) { logger.debug("Album is already in cache : " + directory); } else { @@ -67,6 +70,9 @@ protected void handleDirectoryStart(File directory, int depth, Collection protected void handleFile(File file, int depth, java.util.Collection results) throws IOException { LegacyCache legacyCache = legacyCaches.get(file.getParentFile()); + if( !importer.shouldImport(file.getAbsolutePath())) + return; + if (legacyCache.containsImage(file)) { logger.debug("Image already in cache : " + file); } else { From 4d7e3bc597d96c5b88d1d5600e41dd12cd3ce65c Mon Sep 17 00:00:00 2001 From: mingan <11653241+mingan666@users.noreply.github.com> Date: Sat, 11 Feb 2023 23:17:50 +0100 Subject: [PATCH 5/5] Add Dockerfile --- remotesync-api/Dockerfile | 8 ++++++++ remotesync-api/pom.xml | 14 ++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 remotesync-api/Dockerfile diff --git a/remotesync-api/Dockerfile b/remotesync-api/Dockerfile new file mode 100644 index 0000000..e3426bb --- /dev/null +++ b/remotesync-api/Dockerfile @@ -0,0 +1,8 @@ +FROM eclipse-temurin:17-alpine + +WORKDIR /opt/app +COPY target/lib /opt/app/lib +COPY target/classes /opt/app + +EXPOSE 8080 +ENTRYPOINT ["java","-cp","/opt/app:/opt/app/lib/*","org.piwigo.remotesync.api.Main"] diff --git a/remotesync-api/pom.xml b/remotesync-api/pom.xml index 2aa53d0..97cc146 100644 --- a/remotesync-api/pom.xml +++ b/remotesync-api/pom.xml @@ -160,6 +160,20 @@ + + maven-dependency-plugin + + + install + + copy-dependencies + + + ${project.build.directory}/lib + + + +