Skip to content

Commit

Permalink
🐛 Better impl and tests for URIs; Resolves #421
Browse files Browse the repository at this point in the history
  • Loading branch information
ebullient committed Apr 21, 2024
1 parent 5f31ad6 commit 5ac05f3
Show file tree
Hide file tree
Showing 8 changed files with 158 additions and 112 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,7 @@ private enum ConfigKeys {
useDiceRoller,
exclude,
excludePattern,
fallbackPaths(List.of("fallback-paths")),
from,
fullSource(List.of("convert", "full-source")),
images,
Expand Down Expand Up @@ -502,6 +503,7 @@ static class ImageOptions {
String internalRoot;
Boolean copyInternal;
Boolean copyExternal;
final Map<String, String> fallbackPaths = new HashMap<>();

public ImageOptions() {
}
Expand All @@ -511,6 +513,7 @@ public ImageOptions(ImageOptions images, ImageOptions images2) {
copyExternal = images.copyExternal;
copyInternal = images.copyInternal;
internalRoot = images.internalRoot;
fallbackPaths.putAll(images.fallbackPaths);
}
if (images2 != null) {
copyExternal = images2.copyExternal == null
Expand All @@ -522,6 +525,7 @@ public ImageOptions(ImageOptions images, ImageOptions images2) {
internalRoot = images2.internalRoot == null
? internalRoot
: images2.internalRoot;
fallbackPaths.putAll(images2.fallbackPaths);
}
}

Expand All @@ -532,6 +536,10 @@ public boolean copyExternal() {
public boolean copyInternal() {
return copyInternal != null && copyInternal;
}

public Map<String, String> fallbackPaths() {
return Collections.unmodifiableMap(fallbackPaths);
}
}

@RegisterForReflection
Expand Down
10 changes: 6 additions & 4 deletions src/main/java/dev/ebullient/convert/config/TtrpgConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -95,9 +95,11 @@ public static class ImageRoot {
final String internalImageRoot;
final boolean copyInternal;
final boolean copyExternal;
final Map<String, String> fallbackPaths;

private ImageRoot(String cfgRoot, ImageOptions options) {
this.copyExternal = options.copyExternal();
this.fallbackPaths = options.fallbackPaths();

if (cfgRoot == null) {
this.internalImageRoot = "";
Expand Down Expand Up @@ -139,6 +141,10 @@ private String endWithSlash(String path) {
}
return path.endsWith("/") ? path : path + "/";
}

public String getFallbackPath(String key) {
return fallbackPaths.getOrDefault(key, key);
}
}

public static ImageRoot internalImageRoot() {
Expand All @@ -154,10 +160,6 @@ public static ImageRoot internalImageRoot() {
return root;
}

public static Map<String, String> imageFallbackPaths() {
return activeDSConfig().fallbackImagePaths;
}

public static JsonNode readIndex(String key) {
String file = activeDSConfig().indexes.get(key);
Optional<Path> root = file == null ? Optional.empty() : tui.resolvePath(Path.of(file));
Expand Down
32 changes: 29 additions & 3 deletions src/main/java/dev/ebullient/convert/io/Tui.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.channels.Channels;
Expand Down Expand Up @@ -213,7 +215,7 @@ public void init(CommandSpec spec, boolean debug, boolean verbose, boolean log)
this.debug = debug || log;
this.verbose = verbose;
if (log) {
Path p = Path.of("ttrpg-convert.out");
Path p = Path.of("ttrpg-convert.out.txt");
try {
this.log = new PrintWriter(Files.newOutputStream(p));
VersionProvider vp = new VersionProvider();
Expand Down Expand Up @@ -436,6 +438,28 @@ private void copyImageResource(ImageRef image, Path targetPath) {
}
}

private final static String allowedCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~/";

String escapeUrlImagePath(String url) throws MalformedURLException, UnsupportedEncodingException {
URL urlObject = new URL(url);
String path = urlObject.getPath();

StringBuilder encodedPath = new StringBuilder();
for (char ch : path.toCharArray()) {
if (allowedCharacters.indexOf(ch) == -1) {
byte[] bytes = String.valueOf(ch).getBytes("UTF-8");
for (byte b : bytes) {
encodedPath.append(String.format("%%%02X", b));
}
} else {
encodedPath.append(ch);
}
}

return url.replace(path, encodedPath.toString())
.replace("/imgur.com", "/i.imgur.com");
}

private void copyRemoteImage(ImageRef image, Path targetPath) {
targetPath.getParent().toFile().mkdirs();

Expand All @@ -445,12 +469,14 @@ private void copyRemoteImage(ImageRef image, Path targetPath) {
return;
}
if (!url.startsWith("http") && !url.startsWith("file")) {
errorf("ImageRef %s has invalid URL %s", image.targetFilePath(), url);
errorf("Remote ImageRef %s has invalid URL %s", image.targetFilePath(), url);
return;
}

Tui.instance().debugf("copy image %s %n to %s", url, targetPath);
try {
url = escapeUrlImagePath(url);
Tui.instance().debugf("copy image %s", url);

ReadableByteChannel readableByteChannel = Channels.newChannel(new URL(url).openStream());
try (FileOutputStream fileOutputStream = new FileOutputStream(targetPath.toFile())) {
fileOutputStream.getChannel().transferFrom(readableByteChannel, 0, Long.MAX_VALUE);
Expand Down
85 changes: 37 additions & 48 deletions src/main/java/dev/ebullient/convert/qute/ImageRef.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package dev.ebullient.convert.qute;

import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;

import dev.ebullient.convert.config.TtrpgConfig;
Expand Down Expand Up @@ -43,7 +45,7 @@ public class ImageRef {
final String titleAttr;

private ImageRef(String url, Path sourcePath, Path targetFilePath, String title, String vaultPath, Integer width) {
this.url = url == null ? null : url.replace(" ", "%20"); // catch any remaining spaces
this.url = url;
this.sourcePath = sourcePath;
this.targetFilePath = targetFilePath;
title = title == null
Expand All @@ -59,10 +61,6 @@ private ImageRef(String url, Path sourcePath, Path targetFilePath, String title,
}
this.vaultPath = vaultPath;
this.width = width;

if (url == null && vaultPath == null) {
Tui.instance().errorf("ImageRef (target=%s) has no url or vaultPath", targetFilePath);
}
}

String escape(String s) {
Expand Down Expand Up @@ -208,61 +206,52 @@ public Builder setUrl(String url) {
}

public ImageRef build() {
final ImageRoot imageRoot = TtrpgConfig.internalImageRoot();

if (url != null && !imageRoot.copyExternalToVault()) {
// leave external images alone (referenced as url)
return new ImageRef(url, null, null, title, null, width);
}

if (url == null && sourcePath == null) {
Tui.instance().errorf("ImageRef build for internal image called without url or sourcePath set");
return null;
}
if (relativeTarget == null || vaultRoot == null || rootFilePath == null) {
Tui.instance().errorf("ImageRef build called without target paths set");
return null;

final ImageRoot imageRoot = TtrpgConfig.internalImageRoot();
String sourceUrl = url == null ? sourcePath.toString() : url;

// Check for any URL replacements (to replace a not-found-image with a local one, e.g.)
// replace backslashes with forward slashes
sourceUrl = imageRoot.getFallbackPath(sourceUrl)
.replace('\\', '/');

try {
// Remove escaped characters here (local file paths won't want it)
sourceUrl = java.net.URLDecoder.decode(sourceUrl, StandardCharsets.UTF_8.name());
} catch (UnsupportedEncodingException e) {
Tui.instance().errorf("Error decoding image URL: %s", e.getMessage());
}

Path targetFilePath = rootFilePath.resolve(relativeTarget);
String vaultPath = String.format("%s%s", vaultRoot,
relativeTarget.toString().replace('\\', '/'));

// Escaping spaces is a mess. Remove here (local file paths won't want it)
// It is changed back (from space to %20) if in URL form (file or http)
String remoteUrl = url == null
? sourcePath.toString().replace("%20", " ")
: url;
if (remoteUrl.startsWith("http")) {
remoteUrl = remoteUrl
.replaceAll("^(https?):/+", "$1://")
.replace("/imgur.com", "/i.imgur.com");
} else if (!remoteUrl.startsWith("file:/")) {
remoteUrl = imageRoot.getRootPath() + remoteUrl;
boolean copyToVault = false;

if (sourceUrl.startsWith("http") || sourceUrl.startsWith("file")) {
sourceUrl = sourceUrl.replaceAll("^(https?):/+", "$1://");
copyToVault = imageRoot.copyExternalToVault();
} else if (!sourceUrl.startsWith("file:/")) {
sourceUrl = imageRoot.getRootPath() + sourceUrl;
copyToVault = imageRoot.copyInternalToVault();
}

if (imageRoot.copyInternalToVault() || imageRoot.copyExternalToVault()) {
boolean localTargetSet = relativeTarget != null && vaultRoot != null && rootFilePath != null;
if (localTargetSet && copyToVault) {
Path targetFilePath = rootFilePath.resolve(relativeTarget);
String vaultPath = String.format("%s%s", vaultRoot,
relativeTarget.toString().replace('\\', '/'));

// remote images to be copied into the vault
if (remoteUrl.startsWith("http") || remoteUrl.startsWith("file")) {
String filename = remoteUrl.substring(remoteUrl.lastIndexOf('/') + 1);
if (!filename.contains("%")) {
try {
String encoded = java.net.URLEncoder.encode(filename, "UTF-8")
.replace("+", "%20");
remoteUrl = remoteUrl.replace(filename, encoded);
} catch (java.io.UnsupportedEncodingException e) {
Tui.instance().errorf("Failed to encode filename: %s", filename);
}
}
// also replace any remaining spaces in the path
return new ImageRef(remoteUrl,
null, targetFilePath, title, vaultPath, width);
if (sourceUrl.startsWith("http") || sourceUrl.startsWith("file")) {
return new ImageRef(sourceUrl, null, targetFilePath, title, vaultPath, width);
}
return new ImageRef(null, Path.of(remoteUrl), targetFilePath, title, vaultPath, width);
// local image to be copied into the vault
return new ImageRef(null, Path.of(sourceUrl), targetFilePath, title, vaultPath, width);
}

// remote images are not copied to the vault --> url image ref
return new ImageRef(remoteUrl,
// remote images that are not copied to the vault --> url image ref, no target
return new ImageRef(sourceUrl,
null, null, title, null, width);
}

Expand Down
8 changes: 0 additions & 8 deletions src/main/resources/convertData.json
Original file line number Diff line number Diff line change
Expand Up @@ -332,14 +332,6 @@
"constants": {
"internalImageRoot": "https://raw.githubusercontent.com/5etools-mirror-2/5etools-img/main/"
},
"fallbackImage": {
"img/PSZ/Archon of Redemption.png": "img/PSZ/Archon Of Redemption.png",
"img/bestiary/ERLW/Inspired.png": "img/bestiary/ERLW/Inspired.webp",
"img/bestiary/MTF/Merrenoloth.jpg": "img/bestiary/MTF/Merrenoloth.webp",
"img/bestiary/SDW/Lhammaruntosz.jpg": "img/SDW/Lhammaruntosz.png",
"img/bestiary/VGM/Deep Scion.jpg": "img/bestiary/VGM/Deep Scion.webp",
"img/items/CRCotN/Medal of the Maze.jpg": "img/items/CRCotN/Medal of the Maze.webp"
},
"markerFiles": [
"bestiary/bestiary-mm.json",
"cultsboons.json",
Expand Down
70 changes: 70 additions & 0 deletions src/test/java/dev/ebullient/convert/io/ImgurUrlTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package dev.ebullient.convert.io;

import static org.assertj.core.api.Assertions.assertThat;

import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.nio.file.Path;

import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;

import dev.ebullient.convert.config.TtrpgConfig;
import dev.ebullient.convert.qute.ImageRef;
import dev.ebullient.convert.tools.dnd5e.Tools5eIndex;
import dev.ebullient.convert.tools.dnd5e.Tools5eSources;
import io.quarkus.arc.Arc;
import io.quarkus.test.junit.QuarkusTest;

@QuarkusTest
public class ImgurUrlTest {
protected static Tui tui;
protected static Tools5eIndex index;

@BeforeAll
public static void prepare() {
tui = Arc.container().instance(Tui.class).get();
tui.init(null, true, false);
index = new Tools5eIndex(TtrpgConfig.getConfig());
}

@Test
public void testImgurUrl() throws MalformedURLException, UnsupportedEncodingException {
String input = "https://imgur.com/lQfZ1dF.png";

assertThat(tui.escapeUrlImagePath(input))
.isEqualTo("https://i.imgur.com/lQfZ1dF.png");
}

@Test
public void testAccentedCharacters() throws MalformedURLException, UnsupportedEncodingException {
String input = "https://whatever.com/áé.png?raw=true";

assertThat(tui.escapeUrlImagePath(input))
.isEqualTo("https://whatever.com/%C3%A1%C3%A9.png?raw=true");

Tools5eSources sources = Tools5eSources.findOrTemporary(
Tui.MAPPER.createObjectNode()
.put("name", "Critter")
.put("source", "DMG"));

ImageRef ref = sources.buildTokenImageRef(index,
"https://raw.githubusercontent.com/TheGiddyLimit/homebrew/master/_img/MonsterManualExpanded3/creature/Hill%20Giant%20Warlock%20Of%20Ogrémoch.jpg",
Path.of("something.png"),
false);

assertThat(tui.escapeUrlImagePath(ref.url()))
.isEqualTo(
"https://raw.githubusercontent.com/TheGiddyLimit/homebrew/master/_img/MonsterManualExpanded3/creature/Hill%20Giant%20Warlock%20Of%20Ogr%C3%A9moch.jpg");

ref = sources.buildTokenImageRef(index,
"https://raw.githubusercontent.com/TheGiddyLimit/homebrew/master/_img/MonsterManualExpanded3/creature/token/Stone%20Giant%20Warlock%20Of%20Ogrémoch%20%28Token%29.png",
Path.of("something.png"),
false);

assertThat(tui.escapeUrlImagePath(ref.url()))
.isEqualTo(
"https://raw.githubusercontent.com/TheGiddyLimit/homebrew/master/_img/MonsterManualExpanded3/creature/token/Stone%20Giant%20Warlock%20Of%20Ogr%C3%A9moch%20%28Token%29.png");
}

}
48 changes: 0 additions & 48 deletions src/test/java/dev/ebullient/convert/tools/ImgurUrlTest.java

This file was deleted.

Loading

0 comments on commit 5ac05f3

Please sign in to comment.