From 59654e2ab1f6d3314eff438bf9e30ed6f32e5e74 Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Sun, 26 May 2019 19:54:21 +0200 Subject: [PATCH] Improve cache + jdoc, traces --- src/be/nikiroo/utils/Cache.java | 19 ++++-- src/be/nikiroo/utils/Downloader.java | 58 ++++++++++++------- src/be/nikiroo/utils/StringUtils.java | 18 +++--- .../utils/android/ImageUtilsAndroid.java | 2 +- src/be/nikiroo/utils/ui/ImageUtilsAwt.java | 2 +- 5 files changed, 62 insertions(+), 37 deletions(-) diff --git a/src/be/nikiroo/utils/Cache.java b/src/be/nikiroo/utils/Cache.java index ff2859e..779b194 100644 --- a/src/be/nikiroo/utils/Cache.java +++ b/src/be/nikiroo/utils/Cache.java @@ -281,13 +281,15 @@ public class Cache { * @param uniqueID * a unique ID used to locate the cached resource * + * @return the number of bytes written + * * @throws IOException * in case of I/O error */ - public void save(InputStream in, String uniqueID) throws IOException { + public long save(InputStream in, String uniqueID) throws IOException { File cached = getCached(uniqueID); cached.getParentFile().mkdirs(); - save(in, cached); + return save(in, cached); } /** @@ -298,12 +300,14 @@ public class Cache { * @param url * the {@link URL} used to locate the cached resource * + * @return the number of bytes written + * * @throws IOException * in case of I/O error */ - public void save(InputStream in, URL url) throws IOException { + public long save(InputStream in, URL url) throws IOException { File cached = getCached(url); - save(in, cached); + return save(in, cached); } /** @@ -316,13 +320,16 @@ public class Cache { * @param cached * the cached {@link File} to save to * + * @return the number of bytes written + * * @throws IOException * in case of I/O error */ - private void save(InputStream in, File cached) throws IOException { + private long save(InputStream in, File cached) throws IOException { // We delete AFTER so not to remove the subdir we will use... - IOUtils.write(in, cached); + long bytes = IOUtils.write(in, cached); clean(true, dir, 10); + return bytes; } /** diff --git a/src/be/nikiroo/utils/Downloader.java b/src/be/nikiroo/utils/Downloader.java index f7a5f57..68e4dd7 100644 --- a/src/be/nikiroo/utils/Downloader.java +++ b/src/be/nikiroo/utils/Downloader.java @@ -1,5 +1,6 @@ package be.nikiroo.utils; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStreamWriter; @@ -288,7 +289,6 @@ public class Downloader { "application/x-www-form-urlencoded"); conn.setRequestProperty("Content-Length", Integer.toString(requestData.length())); - conn.setRequestProperty("charset", "utf-8"); } if (oauth != null) { @@ -316,7 +316,8 @@ public class Downloader { conn.connect(); // Check if redirect - // BEWARE! POST data cannot be redirected, so it is ignored here + // BEWARE! POST data cannot be redirected (some webservers complain) for + // HTTP codes 302 and 303 if (conn instanceof HttpURLConnection) { int repCode = 0; try { @@ -328,32 +329,48 @@ public class Downloader { if (repCode / 100 == 3) { String newUrl = conn.getHeaderField("Location"); return open(new URL(newUrl), originalUrl, currentReferer, - cookiesValues, null, getParams, oauth, stable); + cookiesValues, // + (repCode == 302 || repCode == 303) ? null : postParams, // + getParams, oauth, stable); } } - InputStream in = conn.getInputStream(); - if ("gzip".equals(conn.getContentEncoding())) { - in = new GZIPInputStream(in); - } + try { + InputStream in = conn.getInputStream(); + if ("gzip".equals(conn.getContentEncoding())) { + in = new GZIPInputStream(in); + } - if (in != null && cache != null) { - tracer.trace("Save to cache: " + originalUrl); - try { + if (in == null) { + throw new IOException("No InputStream!"); + } + + if (cache != null) { + String size = conn.getContentLengthLong() < 0 ? "unknown size" + : StringUtils.formatNumber(conn.getContentLengthLong()) + + "bytes"; + tracer.trace("Save to cache (" + size + "): " + originalUrl); try { - cache.save(in, originalUrl); - } finally { - in.close(); + try { + long bytes = cache.save(in, originalUrl); + tracer.trace("Saved to cache: " + + StringUtils.formatNumber(bytes) + "bytes"); + } finally { + in.close(); + } + in = cache.load(originalUrl, true, true); + } catch (IOException e) { + tracer.error(new IOException( + "Cannot save URL to cache, will ignore cache: " + + url, e)); } - in = cache.load(originalUrl, true, false); - } catch (IOException e) { - tracer.error(new IOException( - "Cannot save URL to cache, will ignore cache: " + url, - e)); } - } - return in; + return in; + } catch (IOException e) { + throw new IOException(String.format( + "Cannot find %s (current URL: %s)", originalUrl, url), e); + } } /** @@ -381,6 +398,7 @@ public class Downloader { conn.setRequestProperty("User-Agent", UA); conn.setRequestProperty("Accept-Encoding", "gzip"); conn.setRequestProperty("Accept", "*/*"); + conn.setRequestProperty("Charset", "utf-8"); if (currentReferer != null) { conn.setRequestProperty("Referer", currentReferer.toString()); diff --git a/src/be/nikiroo/utils/StringUtils.java b/src/be/nikiroo/utils/StringUtils.java index fa72a4e..b3c1071 100644 --- a/src/be/nikiroo/utils/StringUtils.java +++ b/src/be/nikiroo/utils/StringUtils.java @@ -713,10 +713,10 @@ public class StringUtils { *

* Examples: *

* * @param value @@ -734,10 +734,10 @@ public class StringUtils { *

* Examples (assuming decimalPositions = 1): *

* * @param value @@ -749,7 +749,7 @@ public class StringUtils { */ public static String formatNumber(long value, int decimalPositions) { long userValue = value; - String suffix = ""; + String suffix = " "; long mult = 1; if (value >= 1000000000l) { diff --git a/src/be/nikiroo/utils/android/ImageUtilsAndroid.java b/src/be/nikiroo/utils/android/ImageUtilsAndroid.java index b717989..c2e269c 100644 --- a/src/be/nikiroo/utils/android/ImageUtilsAndroid.java +++ b/src/be/nikiroo/utils/android/ImageUtilsAndroid.java @@ -77,7 +77,7 @@ public class ImageUtilsAndroid extends ImageUtils { Bitmap image = BitmapFactory.decodeStream(stream); if (image == null) { String extra = ""; - if (img.getSize() <= 1024) { + if (img.getSize() <= 2048) { try { extra = ", content: " + new String(img.getData(), "UTF-8"); diff --git a/src/be/nikiroo/utils/ui/ImageUtilsAwt.java b/src/be/nikiroo/utils/ui/ImageUtilsAwt.java index 90d780a..4cf12c0 100644 --- a/src/be/nikiroo/utils/ui/ImageUtilsAwt.java +++ b/src/be/nikiroo/utils/ui/ImageUtilsAwt.java @@ -100,7 +100,7 @@ public class ImageUtilsAwt extends ImageUtils { if (image == null) { String extra = ""; - if (img.getSize() <= 1024) { + if (img.getSize() <= 2048) { try { extra = ", content: " + new String(img.getData(), "UTF-8"); -- 2.27.0