Skip to content

Commit

Permalink
new version 0.0.8:
Browse files Browse the repository at this point in the history 
* cache is now url sensitive
  • Loading branch information
@mhelleboid
mhelleboid committed Nov 11, 2015
1 parent 7dab5e0 commit d32a8fd
Show file tree
Hide file tree
Showing 15 changed files with 188 additions and 104 deletions.
9 changes: 8 additions & 1 deletion remotesync-api/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,8 @@
<archive>
<manifest>
<mainClass>org.piwigo.remotesync.api.Main</mainClass>
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
<addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
</manifest>
</archive>
</configuration>
Expand All @@ -112,12 +114,17 @@
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<configuration>
<finalName>remotesync</finalName>
<outputDirectory>../remotesync/target/</outputDirectory>
<appendAssemblyId>false</appendAssemblyId>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<mainClass>org.piwigo.remotesync.api.Main</mainClass>
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
<addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
</manifest>
</archive>
</configuration>
Expand All @@ -136,7 +143,7 @@
<parent>
<groupId>piwigo</groupId>
<artifactId>remotesync</artifactId>
<version>0.0.7</version>
<version>0.0.8</version>
<relativePath>../remotesync</relativePath>
</parent>
</project>
15 changes: 9 additions & 6 deletions remotesync-api/src/main/java/org/piwigo/remotesync/api/AbstractMain.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,21 +38,24 @@ protected void run(String[] args) {
createConfiguration(parsedSyncConfiguration);

if (help) {
System.out.println("Piwigo Remote Sync : java -jar remotesync.jar");
cmdLineParser.printUsage(System.out);
help(cmdLineParser);
return;
}

start();
} catch (CmdLineException e) {
System.err.println(e.getMessage());
System.err.println("Piwigo Remote Sync : java -jar remotesync.jar");
cmdLineParser.printUsage(System.err);
System.err.println();
System.err.println(" Example: java -jar remotesync.jar" + cmdLineParser.printExample(OptionHandlerFilter.ALL));
System.err.flush();
help(cmdLineParser);
}
}

protected void help(CmdLineParser cmdLineParser) {
System.out.println("Piwigo Remote Sync (version " + getClass().getPackage().getImplementationVersion() + ") : java -jar remotesync.jar");
cmdLineParser.printUsage(System.out);
System.out.println("Example: java -jar remotesync.jar" + cmdLineParser.printExample(OptionHandlerFilter.ALL));
}

protected abstract void start();

@Option(name = "-debug", usage = "enable debug messages")
Expand Down
7 changes: 7 additions & 0 deletions remotesync-api/src/main/java/org/piwigo/remotesync/api/Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.filefilter.AbstractFileFilter;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.piwigo.remotesync.api.conf.ConfigurationUtil;

public class Constants {

Expand All @@ -41,4 +42,10 @@ public boolean accept(File file) {

}

public static final String DIRECTORY_DEFAULT = ConfigurationUtil.INSTANCE.getUserCurrentDirectory().getAbsolutePath();

public static final String CHUNK_SIZE_DEFAULT = "500";

public static final int CHUNK_SIZE_INT_DEFAULT = Integer.parseInt(CHUNK_SIZE_DEFAULT);

}
26 changes: 17 additions & 9 deletions remotesync-api/src/main/java/org/piwigo/remotesync/api/cache/LegacyCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,24 +106,32 @@ public LegacyCache parseFile() {

public LegacyCache parseContent(String content) {
Matcher matcher = ALBUM_PATTERN.matcher(content);
if (matcher.find()) {
albumCacheElement = new AlbumCacheElement();
albumCacheElement.url = matcher.group(1);
albumCacheElement.id = Integer.parseInt(matcher.group(2));
while (matcher.find()) {
if (isSameUrl(matcher.group(1))) {
albumCacheElement = new AlbumCacheElement();
albumCacheElement.url = matcher.group(1);
albumCacheElement.id = Integer.parseInt(matcher.group(2));
}
}

matcher = IMAGE_PATTERN.matcher(content);
while (matcher.find()) {
ImageCacheElement imageCacheElement = new ImageCacheElement();
imageCacheElement.url = matcher.group(1);
imageCacheElement.filePathMD5 = matcher.group(2);
imageCacheElement.id = Integer.parseInt(matcher.group(3));
imagesCache.add(imageCacheElement);
if (isSameUrl(matcher.group(1))) {
ImageCacheElement imageCacheElement = new ImageCacheElement();
imageCacheElement.url = matcher.group(1);
imageCacheElement.filePathMD5 = matcher.group(2);
imageCacheElement.id = Integer.parseInt(matcher.group(3));
imagesCache.add(imageCacheElement);
}
}

return this;
}

public boolean isSameUrl(String otherUrl) {
return url.replaceAll("https?", "").equals(otherUrl.replaceAll("https?", ""));
}

protected void writeToFile(ILegacyCacheElement abstractCacheElement) {
try {
FileUtils.writeStringToFile(cacheFile, abstractCacheElement.writeToString() + "\n", true);
Expand Down
4 changes: 2 additions & 2 deletions remotesync-api/src/main/java/org/piwigo/remotesync/api/client/AbstractClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,13 @@
public abstract class AbstractClient implements IClient {

@Override
public final <T extends BasicResponse> T sendRequest(AbstractRequest<T> request) throws ClientServerException {
public <T extends BasicResponse> T sendRequest(AbstractRequest<T> request) throws ClientServerException {
return doSendRequest(request);
}

@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
public final <T extends BasicResponse> ComposedResponse<T> sendRequest(ComposedRequest<T> composedRequest) throws ClientServerException {
public <T extends BasicResponse> ComposedResponse<T> sendRequest(ComposedRequest<T> composedRequest) throws ClientServerException {
ComposedResponse<T> composedResponse = new ComposedResponse<T>();
Iterator<AbstractRequest<? extends BasicResponse>> iterator = composedRequest.iterator();
while (iterator.hasNext()) {
Expand Down
132 changes: 80 additions & 52 deletions remotesync-api/src/main/java/org/piwigo/remotesync/api/client/WSClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.NotImplementedException;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
Expand All @@ -36,7 +35,10 @@
import org.piwigo.remotesync.api.exception.ClientServerException;
import org.piwigo.remotesync.api.exception.ServerException;
import org.piwigo.remotesync.api.request.AbstractRequest;
import org.piwigo.remotesync.api.request.ComposedRequest;
import org.piwigo.remotesync.api.request.IChunkable;
import org.piwigo.remotesync.api.response.BasicResponse;
import org.piwigo.remotesync.api.response.ComposedResponse;
import org.piwigo.remotesync.api.response.ServerResponse;
import org.piwigo.remotesync.api.xml.PersisterFactory;
import org.slf4j.Logger;
Expand All @@ -48,8 +50,6 @@
* System.out.println(response.getStatusLine().getStatusCode());
* System.out.println(response.getStatusLine().getReasonPhrase());
* System.out.println(response.getStatusLine().toString());
*
* TODO handle proxy
*/
public class WSClient extends AbstractClient {

Expand All @@ -63,34 +63,69 @@ public WSClient(IClientConfiguration clientConfiguration) {
this.clientConfiguration = clientConfiguration;
}

@SuppressWarnings("unchecked")
@Override
public <T extends BasicResponse> T sendRequest(AbstractRequest<T> request) throws ClientServerException {
handleChunkable(request);
return super.sendRequest(request);
}

@Override
public <T extends BasicResponse> ComposedResponse<T> sendRequest(ComposedRequest<T> composedRequest) throws ClientServerException {
handleChunkable(composedRequest);
return super.sendRequest(composedRequest);
}

protected <T extends BasicResponse> void handleChunkable(AbstractRequest<T> request) {
if (request instanceof IChunkable)
((IChunkable) request).setChunkSize(clientConfiguration.getChunkSize());
}

@Override
protected <T extends BasicResponse> T doSendRequest(AbstractRequest<T> request) throws ClientServerException {
checkRequestAuthorization(request);

if (httpClient == null) {
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
String content = getXmlResponse(request);

if (clientConfiguration.getUsesProxy()) {
String proxyUrl = clientConfiguration.getProxyUrl();
int proxyPort = clientConfiguration.getProxyPort();

String proxyUsername = clientConfiguration.getProxyUsername();
String proxyPassword = clientConfiguration.getProxyPassword();
// basic parsing
ServerResponse errorResponse = parseResponse(content, ServerResponse.class, false);

if ("ok".equals(errorResponse.status)) {
// complete parsing
T response = parseResponse(content, request.getReturnType(), true);
response.setXmlContent(content);
return response;
} else if ("fail".equals(errorResponse.status)) {
logger.debug(content);
throw new ServerException(errorResponse.error.toString());
} else {
throw new NotImplementedException();
}
}

if (proxyUsername != null && proxyUsername.length() > 0 && proxyPassword != null && proxyPassword.length() > 0) {
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(new AuthScope(proxyUrl, proxyPort), new UsernamePasswordCredentials(proxyUsername, proxyPassword));
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
}
protected <T extends BasicResponse> String getXmlResponse(AbstractRequest<T> request) throws ClientException, ServerException {
CloseableHttpResponse httpResponse = null;

try {
httpResponse = getHttpResponse(request);

HttpHost proxy = new HttpHost(proxyUrl, proxyPort);
requestConfig = RequestConfig.custom().setProxy(proxy).build();
if (httpResponse.getStatusLine().getStatusCode() != HttpStatus.SC_OK)
throw new ServerException(httpResponse.getStatusLine().getReasonPhrase() + " (code " + httpResponse.getStatusLine().getStatusCode() + ")");

return IOUtils.toString(httpResponse.getEntity().getContent(), "UTF-8");
} catch (Exception e) {
throw new ClientException("Unable to read response content", e);
} finally {
try {
if (httpResponse != null)
httpResponse.close();
} catch (IOException e) {
logger.error("cannot close post", e);
}
httpClient = httpClientBuilder.build();
}
}

CloseableHttpResponse httpResponse = null;
@SuppressWarnings("unchecked")
protected <T extends BasicResponse> CloseableHttpResponse getHttpResponse(AbstractRequest<T> request) throws ClientException {
try {
HttpPost method = new HttpPost(clientConfiguration.getUrl() + "/ws.php");
method.setConfig(requestConfig);
Expand All @@ -115,46 +150,39 @@ else if (value instanceof List) {
}
method.setEntity(multipartEntityBuilder.build());

httpResponse = httpClient.execute(method);
return getHttpClient().execute(method);
} catch (Exception e) {
throw new ClientException("Unable to send request", e);
}
}

if (httpResponse.getStatusLine().getStatusCode() != HttpStatus.SC_OK)
throw new ServerException(httpResponse.getStatusLine().getReasonPhrase() + " (code " + httpResponse.getStatusLine().getStatusCode() + ")");

String content;
try {
content = IOUtils.toString(httpResponse.getEntity().getContent(), "UTF-8");
} catch (Exception e) {
throw new ClientException("Unable to read response content", e);
}

try {
if (httpResponse != null)
httpResponse.close();
} catch (IOException e) {
logger.error("cannot close post", e);
}

// basic parsing
ServerResponse errorResponse = parseResponse(httpResponse, content, ServerResponse.class, false);
protected CloseableHttpClient getHttpClient() {
if (httpClient == null) {
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();

if ("fail".equals(errorResponse.status)) {
logger.debug(content);
throw new ServerException(errorResponse.error.toString());
} else if (!"ok".equals(errorResponse.status))
throw new NotImplementedException();
if (clientConfiguration.getUsesProxy()) {
String proxyUrl = clientConfiguration.getProxyUrl();
int proxyPort = clientConfiguration.getProxyPort();

String proxyUsername = clientConfiguration.getProxyUsername();
String proxyPassword = clientConfiguration.getProxyPassword();

// complete parsing
T response = parseResponse(httpResponse, content, request.getReturnType(), true);
response.setHttpStatusCode(httpResponse.getStatusLine().getStatusCode());
response.setXmlContent(content);
if (proxyUsername != null && proxyUsername.length() > 0 && proxyPassword != null && proxyPassword.length() > 0) {
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(new AuthScope(proxyUrl, proxyPort), new UsernamePasswordCredentials(proxyUsername, proxyPassword));
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
}

return response;
HttpHost proxy = new HttpHost(proxyUrl, proxyPort);
requestConfig = RequestConfig.custom().setProxy(proxy).build();
}
httpClient = httpClientBuilder.build();
}

return httpClient;
}

private <T extends BasicResponse> T parseResponse(HttpResponse httpResponse, String content, Class<T> type, boolean strict) throws ClientException {
protected <T extends BasicResponse> T parseResponse(String content, Class<T> type, boolean strict) throws ClientException {
try {
return (T) PersisterFactory.createPersister().read(type, content, strict);
} catch (Exception e) {
Expand Down
11 changes: 4 additions & 7 deletions remotesync-api/src/main/java/org/piwigo/remotesync/api/conf/SyncConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import java.lang.reflect.Field;

import org.kohsuke.args4j.Option;
import org.piwigo.remotesync.api.Constants;
import org.piwigo.remotesync.api.ISyncConfiguration;
import org.piwigo.remotesync.api.conf.SyncConfigurationValidator.Validator;
import org.piwigo.remotesync.api.conf.SyncConfigurationValidator.ValidatorRequired;
Expand All @@ -22,10 +23,6 @@
import org.simpleframework.xml.convert.Convert;

public class SyncConfiguration implements ISyncConfiguration {
private static final String DIRECTORY_DEFAULT = ConfigurationUtil.INSTANCE.getUserCurrentDirectory().getAbsolutePath();

private static final String CHUNK_SIZE_DEFAULT = "500";

@Element(required = false)
@Option(name = "-url", usage = "remote gallery url")
@Validator(type = ValidatorType.url, required = ValidatorRequired.yes)
Expand All @@ -46,7 +43,7 @@ public class SyncConfiguration implements ISyncConfiguration {
@Element(required = false)
@Option(name = "-dir", usage = "local directory path")
@Validator(type = ValidatorType.dir)
protected String directory = DIRECTORY_DEFAULT;
protected String directory = Constants.DIRECTORY_DEFAULT;

@Element(required = false)
@Option(name = "-proxy", usage = "use proxy")
Expand Down Expand Up @@ -76,7 +73,7 @@ public class SyncConfiguration implements ISyncConfiguration {
@Element(required = false)
@Option(name = "-cs", usage = "chunk size (in Kbytes)")
@Validator(type = ValidatorType.integer)
protected String chunkSize = CHUNK_SIZE_DEFAULT;
protected String chunkSize = Constants.CHUNK_SIZE_DEFAULT;

public String getValue(String fieldName) {
try {
Expand Down Expand Up @@ -180,7 +177,7 @@ public int getChunkSize() {
try {
return Integer.parseInt(chunkSize);
} catch (NumberFormatException e) {
return Integer.parseInt(CHUNK_SIZE_DEFAULT);
return Integer.parseInt(Constants.CHUNK_SIZE_DEFAULT);
}
}

Expand Down
15 changes: 15 additions & 0 deletions remotesync-api/src/main/java/org/piwigo/remotesync/api/request/IChunkable.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
/*******************************************************************************
* Copyright (c) 2014 Matthieu Helleboid.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the GNU Public License v2.0
* which accompanies this distribution, and is available at
* http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
*
* Contributors:
* Matthieu Helleboid - initial API and implementation
******************************************************************************/
package org.piwigo.remotesync.api.request;

public interface IChunkable {
public void setChunkSize(int chunkSize);
}
Loading

0 comments on commit d32a8fd

Please sign in to comment.