From 9b863b20370118c95c6801b73dda951c7e507871 Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Sun, 12 Aug 2018 20:27:23 +0200 Subject: [PATCH] Fix remote timeot (try1) + reverse e621 /post/ --- changelog-fr.md | 5 ++ changelog.md | 5 ++ .../nikiroo/fanfix/library/BasicLibrary.java | 14 +++++- .../nikiroo/fanfix/library/LocalLibrary.java | 9 ++-- .../fanfix/library/RemoteLibraryServer.java | 47 ++++++++++++++++--- src/be/nikiroo/fanfix/supported/E621.java | 33 +++++++++++-- 6 files changed, 97 insertions(+), 16 deletions(-) diff --git a/changelog-fr.md b/changelog-fr.md index 0dcb934..4e3cf4b 100644 --- a/changelog-fr.md +++ b/changelog-fr.md @@ -1,5 +1,10 @@ # Fanfix +# Version WIP + +- e621: les images étaient rangées à l'envers pour les recherches (/post/) +- remote: correction de certains problèmes de timeout + # Version 1.8.0 - FimfictionAPI: les noms des chapitres sont maintenant triés correctement diff --git a/changelog.md b/changelog.md index 2189c64..4a21c10 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,10 @@ # Fanfix +# Version WIP + +- e621: the images were stored in reverse order for searches (/post/) +- remote: fix some timeout issues + # Version 1.8.0 - FimfictionAPI: chapter names are now correctly ordered diff --git a/src/be/nikiroo/fanfix/library/BasicLibrary.java b/src/be/nikiroo/fanfix/library/BasicLibrary.java index 56a5707..129c8fb 100644 --- a/src/be/nikiroo/fanfix/library/BasicLibrary.java +++ b/src/be/nikiroo/fanfix/library/BasicLibrary.java @@ -398,12 +398,24 @@ abstract public class BasicLibrary { * in case of I/O error */ public Story imprt(URL url, Progress pg) throws IOException { + if (pg == null) + pg = new Progress(); + + pg.setMinMax(0, 1000); + Progress pgProcess = new Progress(); + Progress pgSave = new Progress(); + pg.addProgress(pgProcess, 800); + pg.addProgress(pgSave, 200); + BasicSupport support = BasicSupport.getSupport(url); if (support == null) { throw new UnknownHostException("" + url); } - return save(support.process(pg), null); + Story story = save(support.process(pgProcess), pgSave); + pg.done(); + + return story; } /** diff --git a/src/be/nikiroo/fanfix/library/LocalLibrary.java b/src/be/nikiroo/fanfix/library/LocalLibrary.java index bdded01..838c97c 100644 --- a/src/be/nikiroo/fanfix/library/LocalLibrary.java +++ b/src/be/nikiroo/fanfix/library/LocalLibrary.java @@ -105,8 +105,6 @@ public class LocalLibrary extends BasicLibrary { String mess = "no file found for "; MetaData meta = getInfo(luid); - Instance.getTraceHandler().trace("(info is: " + meta + ")"); - File[] files = getStories(pg).get(meta); if (files != null) { mess = "file retrieved for "; @@ -114,7 +112,8 @@ public class LocalLibrary extends BasicLibrary { } Instance.getTraceHandler().trace( - this.getClass().getSimpleName() + ": " + mess + luid); + this.getClass().getSimpleName() + ": " + mess + luid + " (" + + meta.getTitle() + ")"); return file; } @@ -433,8 +432,8 @@ public class LocalLibrary extends BasicLibrary { * @param pg * the optional {@link Progress} * - * @return the list of stories (for each item, the first {@link File} is the info file, the - * second file is the target {@link File}) + * @return the list of stories (for each item, the first {@link File} is the + * info file, the second file is the target {@link File}) */ private synchronized Map getStories(Progress pg) { if (pg == null) { diff --git a/src/be/nikiroo/fanfix/library/RemoteLibraryServer.java b/src/be/nikiroo/fanfix/library/RemoteLibraryServer.java index dae73e4..0378723 100644 --- a/src/be/nikiroo/fanfix/library/RemoteLibraryServer.java +++ b/src/be/nikiroo/fanfix/library/RemoteLibraryServer.java @@ -3,6 +3,7 @@ package be.nikiroo.fanfix.library; import java.io.IOException; import java.net.URL; import java.util.ArrayList; +import java.util.Date; import java.util.List; import be.nikiroo.fanfix.Instance; @@ -106,8 +107,9 @@ public class RemoteLibraryServer extends ServerObject { return "PONG"; } else if ("GET_METADATA".equals(command)) { if ("*".equals(args[0])) { - List metas = Instance.getLibrary().getMetas( - createPgForwarder(action)); + Progress pg = createPgForwarder(action); + List metas = Instance.getLibrary().getMetas(pg); + forcePgDoneSent(pg); return metas.toArray(new MetaData[] {}); } @@ -142,8 +144,10 @@ public class RemoteLibraryServer extends ServerObject { Instance.getLibrary().save(story, (String) args[0], null); return story.getMeta().getLuid(); } else if ("IMPORT".equals(command)) { + Progress pg = createPgForwarder(action); Story story = Instance.getLibrary().imprt( - new URL((String) args[0]), createPgForwarder(action)); + new URL((String) args[0]), pg); + forcePgDoneSent(pg); return story.getMeta().getLuid(); } else if ("DELETE_STORY".equals(command)) { Instance.getLibrary().delete((String) args[0]); @@ -155,8 +159,10 @@ public class RemoteLibraryServer extends ServerObject { Instance.getLibrary().setSourceCover((String) args[0], (String) args[1]); } else if ("CHANGE_SOURCE".equals(command)) { + Progress pg = createPgForwarder(action); Instance.getLibrary().changeSource((String) args[0], - (String) args[1], createPgForwarder(action)); + (String) args[1], pg); + forcePgDoneSent(pg); } else if ("EXIT".equals(command)) { stop(0, false); } @@ -263,8 +269,16 @@ public class RemoteLibraryServer extends ServerObject { */ private static Progress createPgForwarder( final ConnectActionServerObject action) { - final Progress pg = new Progress(); + final Boolean[] isDoneForwarded = new Boolean[] { false }; + final Progress pg = new Progress() { + @Override + public boolean isDone() { + return isDoneForwarded[0]; + } + }; + final Integer[] p = new Integer[] { -1, -1, -1 }; + final Long[] lastTime = new Long[] { new Date().getTime() }; pg.addProgressListener(new ProgressListener() { @Override public void progress(Progress progress, String name) { @@ -274,8 +288,11 @@ public class RemoteLibraryServer extends ServerObject { + (int) Math.round(pg.getRelativeProgress() * (max - min)); - // Do not re-send the same value twice over the wire - if (p[0] != min || p[1] != max || p[2] != relativeProgress) { + // Do not re-send the same value twice over the wire, + // unless more than 2 seconds have elapsed (to maintain the + // connection) + if ((p[0] != min || p[1] != max || p[2] != relativeProgress) + || (new Date().getTime() - lastTime[0] > 2000)) { p[0] = min; p[1] = max; p[2] = relativeProgress; @@ -286,10 +303,26 @@ public class RemoteLibraryServer extends ServerObject { } catch (Exception e) { Instance.getTraceHandler().error(e); } + + isDoneForwarded[0] = pg.isDone(); + lastTime[0] = new Date().getTime(); } } }); return pg; } + + // with 30 seconds timeout + private static void forcePgDoneSent(Progress pg) { + long start = new Date().getTime(); + pg.done(); + while (!pg.isDone() && new Date().getTime() - start < 30000) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Instance.getTraceHandler().error(e); + } + } + } } diff --git a/src/be/nikiroo/fanfix/supported/E621.java b/src/be/nikiroo/fanfix/supported/E621.java index 407ec70..d6cbad4 100644 --- a/src/be/nikiroo/fanfix/supported/E621.java +++ b/src/be/nikiroo/fanfix/supported/E621.java @@ -3,11 +3,14 @@ package be.nikiroo.fanfix.supported; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.net.URLEncoder; import java.util.AbstractMap; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.AbstractMap.SimpleEntry; @@ -284,8 +287,8 @@ class E621 extends BasicSupport_Deprecated { try { if (getLine(pageI, "No posts matched your search.", 0) != null) break; - urls.add(new AbstractMap.SimpleEntry(Integer - .toString(i), url)); + urls.add(new AbstractMap.SimpleEntry("Page " + + Integer.toString(i), url)); } finally { pageI.close(); } @@ -294,6 +297,8 @@ class E621 extends BasicSupport_Deprecated { } } + // They are sorted in reverse order on the website + Collections.reverse(urls); return urls; } @@ -371,11 +376,33 @@ class E621 extends BasicSupport_Deprecated { return builder.toString(); } + @Override + protected URL getCanonicalUrl(URL source) { + if (isSearch(source)) { + // /post?tags=tag1+tag2 -> ../post/index/1/tag1%32tag2 + String key = "post?tags="; + if (source.toString().contains(key)) { + int pos = source.toString().indexOf(key); + String tags = source.toString().substring(pos + key.length()); + tags = tags.replace("+", "%32"); + try { + return new URL(source.toString().substring(0, pos) + + "post/index/1/" + tags); + } catch (MalformedURLException e) { + Instance.getTraceHandler().error(e); + } + } + } + return super.getCanonicalUrl(source); + } + private boolean isPool(URL url) { return url.getPath().startsWith("/pool/"); } private boolean isSearch(URL url) { - return url.getPath().startsWith("/post/index/"); + return url.getPath().startsWith("/post/index/") + || (url.getPath().equals("/post") && url.getQuery().startsWith( + "tags=")); } } -- 2.27.0