c99a0fbebefc1a2fef30d6f83d8c0d080303b5ef
[nikiroo-utils.git] / src / be / nikiroo / fanfix / library / LocalLibrary.java
1 package be.nikiroo.fanfix.library;
2
3 import java.io.File;
4 import java.io.FileFilter;
5 import java.io.FileInputStream;
6 import java.io.FileNotFoundException;
7 import java.io.IOException;
8 import java.io.InputStream;
9 import java.util.ArrayList;
10 import java.util.HashMap;
11 import java.util.List;
12 import java.util.Map;
13
14 import be.nikiroo.fanfix.Instance;
15 import be.nikiroo.fanfix.bundles.Config;
16 import be.nikiroo.fanfix.bundles.ConfigBundle;
17 import be.nikiroo.fanfix.bundles.UiConfigBundle;
18 import be.nikiroo.fanfix.data.MetaData;
19 import be.nikiroo.fanfix.data.Story;
20 import be.nikiroo.fanfix.output.BasicOutput;
21 import be.nikiroo.fanfix.output.BasicOutput.OutputType;
22 import be.nikiroo.fanfix.output.InfoCover;
23 import be.nikiroo.fanfix.supported.InfoReader;
24 import be.nikiroo.utils.IOUtils;
25 import be.nikiroo.utils.Image;
26 import be.nikiroo.utils.Progress;
27 import be.nikiroo.utils.StringUtils;
28
29 /**
30 * This {@link BasicLibrary} will store the stories locally on disk.
31 *
32 * @author niki
33 */
34 public class LocalLibrary extends BasicLibrary {
35 private int lastId;
36 private Map<MetaData, File[]> stories; // Files: [ infoFile, TargetFile ]
37 private Map<String, Image> sourceCovers;
38 private Map<String, Image> authorCovers;
39
40 private File baseDir;
41 private OutputType text;
42 private OutputType image;
43
44 /**
45 * Create a new {@link LocalLibrary} with the given back-end directory.
46 *
47 * @param baseDir the directory where to find the {@link Story} objects
48 * @param config the configuration used to know which kind of default
49 * {@link OutputType} to use for images and non-images stories
50 */
51 public LocalLibrary(File baseDir, ConfigBundle config) {
52 this(baseDir, //
53 config.getString(Config.FILE_FORMAT_NON_IMAGES_DOCUMENT_TYPE),
54 config.getString(Config.FILE_FORMAT_IMAGES_DOCUMENT_TYPE), false);
55 }
56
57 /**
58 * Create a new {@link LocalLibrary} with the given back-end directory.
59 *
60 * @param baseDir
61 * the directory where to find the {@link Story} objects
62 * @param text
63 * the {@link OutputType} to use for non-image documents
64 * @param image
65 * the {@link OutputType} to use for image documents
66 * @param defaultIsHtml
67 * if the given text or image is invalid, use HTML by default (if
68 * not, it will be INFO_TEXT/CBZ by default)
69 */
70 public LocalLibrary(File baseDir, String text, String image,
71 boolean defaultIsHtml) {
72 this(baseDir, OutputType.valueOfAllOkUC(text,
73 defaultIsHtml ? OutputType.HTML : OutputType.INFO_TEXT),
74 OutputType.valueOfAllOkUC(image,
75 defaultIsHtml ? OutputType.HTML : OutputType.CBZ));
76 }
77
78 /**
79 * Create a new {@link LocalLibrary} with the given back-end directory.
80 *
81 * @param baseDir
82 * the directory where to find the {@link Story} objects
83 * @param text
84 * the {@link OutputType} to use for non-image documents
85 * @param image
86 * the {@link OutputType} to use for image documents
87 */
88 public LocalLibrary(File baseDir, OutputType text, OutputType image) {
89 this.baseDir = baseDir;
90 this.text = text;
91 this.image = image;
92
93 this.lastId = 0;
94 this.stories = null;
95 this.sourceCovers = null;
96
97 baseDir.mkdirs();
98 }
99
100 @Override
101 protected List<MetaData> getMetas(Progress pg) {
102 return new ArrayList<MetaData>(getStories(pg).keySet());
103 }
104
105 @Override
106 public File getFile(String luid, Progress pg) throws IOException {
107 Instance.getInstance().getTraceHandler().trace(this.getClass().getSimpleName() + ": get file for " + luid);
108
109 File file = null;
110 String mess = "no file found for ";
111
112 MetaData meta = getInfo(luid);
113 File[] files = getStories(pg).get(meta);
114 if (files != null) {
115 mess = "file retrieved for ";
116 file = files[1];
117 }
118
119 Instance.getInstance().getTraceHandler()
120 .trace(this.getClass().getSimpleName() + ": " + mess + luid + " (" + meta.getTitle() + ")");
121
122 return file;
123 }
124
125 @Override
126 public Image getCover(String luid) throws IOException {
127 MetaData meta = getInfo(luid);
128 if (meta != null) {
129 if (meta.getCover() != null) {
130 return meta.getCover();
131 }
132
133 File[] files = getStories(null).get(meta);
134 if (files != null) {
135 File infoFile = files[0];
136
137 try {
138 meta = InfoReader.readMeta(infoFile, true);
139 return meta.getCover();
140 } catch (IOException e) {
141 Instance.getInstance().getTraceHandler().error(e);
142 }
143 }
144 }
145
146 return null;
147 }
148
149 @Override
150 protected synchronized void updateInfo(MetaData meta) {
151 invalidateInfo();
152 }
153
154 @Override
155 protected void invalidateInfo(String luid) {
156 stories = null;
157 sourceCovers = null;
158 }
159
160 @Override
161 protected synchronized int getNextId() {
162 getStories(null); // make sure lastId is set
163 return ++lastId;
164 }
165
166 @Override
167 protected void doDelete(String luid) throws IOException {
168 for (File file : getRelatedFiles(luid)) {
169 // TODO: throw an IOException if we cannot delete the files?
170 IOUtils.deltree(file);
171 file.getParentFile().delete();
172 }
173 }
174
175 @Override
176 protected Story doSave(Story story, Progress pg) throws IOException {
177 MetaData meta = story.getMeta();
178
179 File expectedTarget = getExpectedFile(meta);
180 expectedTarget.getParentFile().mkdirs();
181
182 BasicOutput it = BasicOutput.getOutput(getOutputType(meta), true, true);
183 it.process(story, expectedTarget.getPath(), pg);
184
185 return story;
186 }
187
188 @Override
189 protected synchronized void saveMeta(MetaData meta, Progress pg)
190 throws IOException {
191 File newDir = getExpectedDir(meta.getSource());
192 if (!newDir.exists()) {
193 newDir.mkdirs();
194 }
195
196 List<File> relatedFiles = getRelatedFiles(meta.getLuid());
197 for (File relatedFile : relatedFiles) {
198 // TODO: this is not safe at all.
199 // We should copy all the files THEN delete them
200 // Maybe also adding some rollback cleanup if possible
201 if (relatedFile.getName().endsWith(".info")) {
202 try {
203 String name = relatedFile.getName().replaceFirst(
204 "\\.info$", "");
205 relatedFile.delete();
206 InfoCover.writeInfo(newDir, name, meta);
207 relatedFile.getParentFile().delete();
208 } catch (IOException e) {
209 Instance.getInstance().getTraceHandler().error(e);
210 }
211 } else {
212 relatedFile.renameTo(new File(newDir, relatedFile.getName()));
213 relatedFile.getParentFile().delete();
214 }
215 }
216
217 invalidateInfo();
218 }
219
220 @Override
221 public synchronized Image getCustomSourceCover(String source) {
222 if (sourceCovers == null) {
223 sourceCovers = new HashMap<String, Image>();
224 }
225
226 Image img = sourceCovers.get(source);
227 if (img != null) {
228 return img;
229 }
230
231 File coverDir = getExpectedDir(source);
232 if (coverDir.isDirectory()) {
233 File cover = new File(coverDir, ".cover.png");
234 if (cover.exists()) {
235 InputStream in;
236 try {
237 in = new FileInputStream(cover);
238 try {
239 sourceCovers.put(source, new Image(in));
240 } finally {
241 in.close();
242 }
243 } catch (FileNotFoundException e) {
244 e.printStackTrace();
245 } catch (IOException e) {
246 Instance.getInstance().getTraceHandler()
247 .error(new IOException("Cannot load the existing custom source cover: " + cover, e));
248 }
249 }
250 }
251
252 return sourceCovers.get(source);
253 }
254
255 @Override
256 public synchronized Image getCustomAuthorCover(String author) {
257 if (authorCovers == null) {
258 authorCovers = new HashMap<String, Image>();
259 }
260
261 Image img = authorCovers.get(author);
262 if (img != null) {
263 return img;
264 }
265
266 File cover = getAuthorCoverFile(author);
267 if (cover.exists()) {
268 InputStream in;
269 try {
270 in = new FileInputStream(cover);
271 try {
272 authorCovers.put(author, new Image(in));
273 } finally {
274 in.close();
275 }
276 } catch (FileNotFoundException e) {
277 e.printStackTrace();
278 } catch (IOException e) {
279 Instance.getInstance().getTraceHandler()
280 .error(new IOException("Cannot load the existing custom author cover: " + cover, e));
281 }
282 }
283
284 return authorCovers.get(author);
285 }
286
287 @Override
288 public void setSourceCover(String source, String luid) throws IOException {
289 setSourceCover(source, getCover(luid));
290 }
291
292 @Override
293 public void setAuthorCover(String author, String luid) throws IOException {
294 setAuthorCover(author, getCover(luid));
295 }
296
297 /**
298 * Set the source cover to the given story cover.
299 *
300 * @param source
301 * the source to change
302 * @param coverImage
303 * the cover image
304 */
305 synchronized void setSourceCover(String source, Image coverImage) {
306 File dir = getExpectedDir(source);
307 dir.mkdirs();
308 File cover = new File(dir, ".cover");
309 try {
310 Instance.getInstance().getCache().saveAsImage(coverImage, cover, true);
311 if (sourceCovers != null) {
312 sourceCovers.put(source, coverImage);
313 }
314 } catch (IOException e) {
315 Instance.getInstance().getTraceHandler().error(e);
316 }
317 }
318
319 /**
320 * Set the author cover to the given story cover.
321 *
322 * @param author
323 * the author to change
324 * @param coverImage
325 * the cover image
326 */
327 synchronized void setAuthorCover(String author, Image coverImage) {
328 File cover = getAuthorCoverFile(author);
329 cover.getParentFile().mkdirs();
330 try {
331 Instance.getInstance().getCache().saveAsImage(coverImage, cover, true);
332 if (authorCovers != null) {
333 authorCovers.put(author, coverImage);
334 }
335 } catch (IOException e) {
336 Instance.getInstance().getTraceHandler().error(e);
337 }
338 }
339
340 @Override
341 public void imprt(BasicLibrary other, String luid, Progress pg)
342 throws IOException {
343 if (pg == null) {
344 pg = new Progress();
345 }
346
347 // Check if we can simply copy the files instead of the whole process
348 if (other instanceof LocalLibrary) {
349 LocalLibrary otherLocalLibrary = (LocalLibrary) other;
350
351 MetaData meta = otherLocalLibrary.getInfo(luid);
352 String expectedType = ""
353 + (meta != null && meta.isImageDocument() ? image : text);
354 if (meta != null && meta.getType().equals(expectedType)) {
355 File from = otherLocalLibrary.getExpectedDir(meta.getSource());
356 File to = this.getExpectedDir(meta.getSource());
357 List<File> relatedFiles = otherLocalLibrary
358 .getRelatedFiles(luid);
359 if (!relatedFiles.isEmpty()) {
360 pg.setMinMax(0, relatedFiles.size());
361 }
362
363 for (File relatedFile : relatedFiles) {
364 File target = new File(relatedFile.getAbsolutePath()
365 .replace(from.getAbsolutePath(),
366 to.getAbsolutePath()));
367 if (!relatedFile.equals(target)) {
368 target.getParentFile().mkdirs();
369 InputStream in = null;
370 try {
371 in = new FileInputStream(relatedFile);
372 IOUtils.write(in, target);
373 } catch (IOException e) {
374 if (in != null) {
375 try {
376 in.close();
377 } catch (Exception ee) {
378 }
379 }
380
381 pg.done();
382 throw e;
383 }
384 }
385
386 pg.add(1);
387 }
388
389 invalidateInfo();
390 pg.done();
391 return;
392 }
393 }
394
395 super.imprt(other, luid, pg);
396 }
397
398 /**
399 * Return the {@link OutputType} for this {@link Story}.
400 *
401 * @param meta
402 * the {@link Story} {@link MetaData}
403 *
404 * @return the type
405 */
406 private OutputType getOutputType(MetaData meta) {
407 if (meta != null && meta.isImageDocument()) {
408 return image;
409 }
410
411 return text;
412 }
413
414 /**
415 * Return the default {@link OutputType} for this kind of {@link Story}.
416 *
417 * @param imageDocument
418 * TRUE for images document, FALSE for text documents
419 *
420 * @return the type
421 */
422 public String getOutputType(boolean imageDocument) {
423 if (imageDocument) {
424 return image.toString();
425 }
426
427 return text.toString();
428 }
429
430 /**
431 * Get the target {@link File} related to the given <tt>.info</tt>
432 * {@link File} and {@link MetaData}.
433 *
434 * @param meta
435 * the meta
436 * @param infoFile
437 * the <tt>.info</tt> {@link File}
438 *
439 * @return the target {@link File}
440 */
441 private File getTargetFile(MetaData meta, File infoFile) {
442 // Replace .info with whatever is needed:
443 String path = infoFile.getPath();
444 path = path.substring(0, path.length() - ".info".length());
445 String newExt = getOutputType(meta).getDefaultExtension(true);
446
447 return new File(path + newExt);
448 }
449
450 /**
451 * The target (full path) where the {@link Story} related to this
452 * {@link MetaData} should be located on disk for a new {@link Story}.
453 *
454 * @param key
455 * the {@link Story} {@link MetaData}
456 *
457 * @return the target
458 */
459 private File getExpectedFile(MetaData key) {
460 String title = key.getTitle();
461 if (title == null) {
462 title = "";
463 }
464 title = title.replaceAll("[^a-zA-Z0-9._+-]", "_");
465 if (title.length() > 40) {
466 title = title.substring(0, 40);
467 }
468 return new File(getExpectedDir(key.getSource()), key.getLuid() + "_"
469 + title);
470 }
471
472 /**
473 * The directory (full path) where the new {@link Story} related to this
474 * {@link MetaData} should be located on disk.
475 *
476 * @param source
477 * the type (source)
478 *
479 * @return the target directory
480 */
481 private File getExpectedDir(String source) {
482 String sanitizedSource = source.replaceAll("[^a-zA-Z0-9._+/-]", "_");
483
484 while (sanitizedSource.startsWith("/")
485 || sanitizedSource.startsWith("_")) {
486 if (sanitizedSource.length() > 1) {
487 sanitizedSource = sanitizedSource.substring(1);
488 } else {
489 sanitizedSource = "";
490 }
491 }
492
493 sanitizedSource = sanitizedSource.replace("/", File.separator);
494
495 if (sanitizedSource.isEmpty()) {
496 sanitizedSource = "_EMPTY";
497 }
498
499 return new File(baseDir, sanitizedSource);
500 }
501
502 /**
503 * Return the full path to the file to use for the custom cover of this
504 * author.
505 * <p>
506 * One or more of the parent directories <b>MAY</b> not exist.
507 *
508 * @param author
509 * the author
510 *
511 * @return the custom cover file
512 */
513 private File getAuthorCoverFile(String author) {
514 File aDir = new File(baseDir, "_AUTHORS");
515 String hash = StringUtils.getMd5Hash(author);
516 String ext = Instance.getInstance().getConfig().getString(Config.FILE_FORMAT_IMAGE_FORMAT_COVER);
517 return new File(aDir, hash + "." + ext.toLowerCase());
518 }
519
520 /**
521 * Return the list of files/directories on disk for this {@link Story}.
522 * <p>
523 * If the {@link Story} is not found, and empty list is returned.
524 *
525 * @param luid
526 * the {@link Story} LUID
527 *
528 * @return the list of {@link File}s
529 *
530 * @throws IOException
531 * if the {@link Story} was not found
532 */
533 private List<File> getRelatedFiles(String luid) throws IOException {
534 List<File> files = new ArrayList<File>();
535
536 MetaData meta = getInfo(luid);
537 if (meta == null) {
538 throw new IOException("Story not found: " + luid);
539 }
540
541 File infoFile = getStories(null).get(meta)[0];
542 File targetFile = getStories(null).get(meta)[1];
543
544 files.add(infoFile);
545 files.add(targetFile);
546
547 String readerExt = getOutputType(meta).getDefaultExtension(true);
548 String fileExt = getOutputType(meta).getDefaultExtension(false);
549
550 String path = targetFile.getAbsolutePath();
551 if (readerExt != null && !readerExt.equals(fileExt)) {
552 path = path.substring(0, path.length() - readerExt.length())
553 + fileExt;
554 File relatedFile = new File(path);
555
556 if (relatedFile.exists()) {
557 files.add(relatedFile);
558 }
559 }
560
561 String coverExt = "."
562 + Instance.getInstance().getConfig().getString(Config.FILE_FORMAT_IMAGE_FORMAT_COVER).toLowerCase();
563 File coverFile = new File(path + coverExt);
564 if (!coverFile.exists()) {
565 coverFile = new File(path.substring(0,
566 path.length() - fileExt.length())
567 + coverExt);
568 }
569
570 if (coverFile.exists()) {
571 files.add(coverFile);
572 }
573
574 return files;
575 }
576
577 /**
578 * Fill the list of stories by reading the content of the local directory
579 * {@link LocalLibrary#baseDir}.
580 * <p>
581 * Will use a cached list when possible (see
582 * {@link BasicLibrary#invalidateInfo()}).
583 *
584 * @param pg
585 * the optional {@link Progress}
586 *
587 * @return the list of stories (for each item, the first {@link File} is the
588 * info file, the second file is the target {@link File})
589 */
590 private synchronized Map<MetaData, File[]> getStories(Progress pg) {
591 if (pg == null) {
592 pg = new Progress();
593 } else {
594 pg.setMinMax(0, 100);
595 }
596
597 if (stories == null) {
598 stories = new HashMap<MetaData, File[]>();
599
600 lastId = 0;
601
602 File[] dirs = baseDir.listFiles(new FileFilter() {
603 @Override
604 public boolean accept(File file) {
605 return file != null && file.isDirectory();
606 }
607 });
608
609 if (dirs != null) {
610 Progress pgDirs = new Progress(0, 100 * dirs.length);
611 pg.addProgress(pgDirs, 100);
612
613 for (File dir : dirs) {
614 Progress pgFiles = new Progress();
615 pgDirs.addProgress(pgFiles, 100);
616 pgDirs.setName("Loading from: " + dir.getName());
617
618 addToStories(pgFiles, dir);
619
620 pgFiles.setName(null);
621 }
622
623 pgDirs.setName("Loading directories");
624 }
625 }
626
627 pg.done();
628 return stories;
629 }
630
631 private void addToStories(Progress pgFiles, File dir) {
632 File[] infoFilesAndSubdirs = dir.listFiles(new FileFilter() {
633 @Override
634 public boolean accept(File file) {
635 boolean info = file != null && file.isFile()
636 && file.getPath().toLowerCase().endsWith(".info");
637 boolean dir = file != null && file.isDirectory();
638 boolean isExpandedHtml = new File(file, "index.html").isFile();
639 return info || (dir && !isExpandedHtml);
640 }
641 });
642
643 if (pgFiles != null) {
644 pgFiles.setMinMax(0, infoFilesAndSubdirs.length);
645 }
646
647 for (File infoFileOrSubdir : infoFilesAndSubdirs) {
648 if (pgFiles != null) {
649 pgFiles.setName(infoFileOrSubdir.getName());
650 }
651
652 if (infoFileOrSubdir.isDirectory()) {
653 addToStories(null, infoFileOrSubdir);
654 } else {
655 try {
656 MetaData meta = InfoReader
657 .readMeta(infoFileOrSubdir, false);
658 try {
659 int id = Integer.parseInt(meta.getLuid());
660 if (id > lastId) {
661 lastId = id;
662 }
663
664 stories.put(meta, new File[] { infoFileOrSubdir,
665 getTargetFile(meta, infoFileOrSubdir) });
666 } catch (Exception e) {
667 // not normal!!
668 throw new IOException("Cannot understand the LUID of "
669 + infoFileOrSubdir + ": " + meta.getLuid(), e);
670 }
671 } catch (IOException e) {
672 // We should not have not-supported files in the
673 // library
674 Instance.getInstance().getTraceHandler()
675 .error(new IOException("Cannot load file from library: " + infoFileOrSubdir, e));
676 }
677 }
678
679 if (pgFiles != null) {
680 pgFiles.add(1);
681 }
682 }
683 }
684 }