merge from master
[nikiroo-utils.git] / library / LocalLibrary.java
... / ...
CommitLineData
1package be.nikiroo.fanfix.library;
2
3import java.io.File;
4import java.io.FileFilter;
5import java.io.FileInputStream;
6import java.io.FileNotFoundException;
7import java.io.IOException;
8import java.io.InputStream;
9import java.util.ArrayList;
10import java.util.HashMap;
11import java.util.List;
12import java.util.Map;
13
14import be.nikiroo.fanfix.Instance;
15import be.nikiroo.fanfix.bundles.Config;
16import be.nikiroo.fanfix.bundles.ConfigBundle;
17import be.nikiroo.fanfix.data.MetaData;
18import be.nikiroo.fanfix.data.Story;
19import be.nikiroo.fanfix.output.BasicOutput;
20import be.nikiroo.fanfix.output.BasicOutput.OutputType;
21import be.nikiroo.fanfix.output.InfoCover;
22import be.nikiroo.fanfix.supported.InfoReader;
23import be.nikiroo.utils.HashUtils;
24import be.nikiroo.utils.IOUtils;
25import be.nikiroo.utils.Image;
26import be.nikiroo.utils.Progress;
27
28/**
29 * This {@link BasicLibrary} will store the stories locally on disk.
30 *
31 * @author niki
32 */
33public class LocalLibrary extends BasicLibrary {
34 private int lastId;
35 private Object lock = new Object();
36 private Map<MetaData, File[]> stories; // Files: [ infoFile, TargetFile ]
37 private Map<String, Image> sourceCovers;
38 private Map<String, Image> authorCovers;
39
40 private File baseDir;
41 private OutputType text;
42 private OutputType image;
43
44 /**
45 * Create a new {@link LocalLibrary} with the given back-end directory.
46 *
47 * @param baseDir
48 * the directory where to find the {@link Story} objects
49 * @param config
50 * the configuration used to know which kind of default
51 * {@link OutputType} to use for images and non-images stories
52 */
53 public LocalLibrary(File baseDir, ConfigBundle config) {
54 this(baseDir, //
55 config.getString(Config.FILE_FORMAT_NON_IMAGES_DOCUMENT_TYPE),
56 config.getString(Config.FILE_FORMAT_IMAGES_DOCUMENT_TYPE),
57 false);
58 }
59
60 /**
61 * Create a new {@link LocalLibrary} with the given back-end directory.
62 *
63 * @param baseDir
64 * the directory where to find the {@link Story} objects
65 * @param text
66 * the {@link OutputType} to use for non-image documents
67 * @param image
68 * the {@link OutputType} to use for image documents
69 * @param defaultIsHtml
70 * if the given text or image is invalid, use HTML by default (if
71 * not, it will be INFO_TEXT/CBZ by default)
72 */
73 public LocalLibrary(File baseDir, String text, String image,
74 boolean defaultIsHtml) {
75 this(baseDir,
76 OutputType.valueOfAllOkUC(text,
77 defaultIsHtml ? OutputType.HTML : OutputType.INFO_TEXT),
78 OutputType.valueOfAllOkUC(image,
79 defaultIsHtml ? OutputType.HTML : OutputType.CBZ));
80 }
81
82 /**
83 * Create a new {@link LocalLibrary} with the given back-end directory.
84 *
85 * @param baseDir
86 * the directory where to find the {@link Story} objects
87 * @param text
88 * the {@link OutputType} to use for non-image documents
89 * @param image
90 * the {@link OutputType} to use for image documents
91 */
92 public LocalLibrary(File baseDir, OutputType text, OutputType image) {
93 this.baseDir = baseDir;
94 this.text = text;
95 this.image = image;
96
97 this.lastId = 0;
98 this.stories = null;
99 this.sourceCovers = null;
100
101 baseDir.mkdirs();
102 }
103
104 @Override
105 protected List<MetaData> getMetas(Progress pg) {
106 return new ArrayList<MetaData>(getStories(pg).keySet());
107 }
108
109 @Override
110 public File getFile(String luid, Progress pg) throws IOException {
111 Instance.getInstance().getTraceHandler().trace(
112 this.getClass().getSimpleName() + ": get file for " + luid);
113
114 File file = null;
115 String mess = "no file found for ";
116
117 MetaData meta = getInfo(luid);
118 if (meta != null) {
119 File[] files = getStories(pg).get(meta);
120 if (files != null) {
121 mess = "file retrieved for ";
122 file = files[1];
123 }
124 }
125
126 Instance.getInstance().getTraceHandler()
127 .trace(this.getClass().getSimpleName() + ": " + mess + luid
128 + " (" + meta.getTitle() + ")");
129
130 return file;
131 }
132
133 @Override
134 public Image getCover(String luid) throws IOException {
135 MetaData meta = getInfo(luid);
136 if (meta != null) {
137 if (meta.getCover() != null) {
138 return meta.getCover();
139 }
140
141 File[] files = getStories(null).get(meta);
142 if (files != null) {
143 File infoFile = files[0];
144
145 try {
146 meta = InfoReader.readMeta(infoFile, true);
147 return meta.getCover();
148 } catch (IOException e) {
149 Instance.getInstance().getTraceHandler().error(e);
150 }
151 }
152 }
153
154 return null;
155 }
156
157 @Override
158 protected void updateInfo(MetaData meta) {
159 invalidateInfo();
160 }
161
162 @Override
163 protected void invalidateInfo(String luid) {
164 synchronized (lock) {
165 stories = null;
166 sourceCovers = null;
167 }
168 }
169
170 @Override
171 protected String getNextId() {
172 getStories(null); // make sure lastId is set
173
174 synchronized (lock) {
175 return String.format("%03d", ++lastId);
176 }
177 }
178
179 @Override
180 protected void doDelete(String luid) throws IOException {
181 for (File file : getRelatedFiles(luid)) {
182 // TODO: throw an IOException if we cannot delete the files?
183 IOUtils.deltree(file);
184 file.getParentFile().delete();
185 }
186 }
187
188 @Override
189 protected Story doSave(Story story, Progress pg) throws IOException {
190 MetaData meta = story.getMeta();
191
192 File expectedTarget = getExpectedFile(meta);
193 expectedTarget.getParentFile().mkdirs();
194
195 BasicOutput it = BasicOutput.getOutput(getOutputType(meta), true, true);
196 it.process(story, expectedTarget.getPath(), pg);
197
198 return story;
199 }
200
201 @Override
202 protected synchronized void saveMeta(MetaData meta, Progress pg)
203 throws IOException {
204 File newDir = getExpectedDir(meta.getSource());
205 if (!newDir.exists()) {
206 newDir.mkdirs();
207 }
208
209 List<File> relatedFiles = getRelatedFiles(meta.getLuid());
210 for (File relatedFile : relatedFiles) {
211 // TODO: this is not safe at all.
212 // We should copy all the files THEN delete them
213 // Maybe also adding some rollback cleanup if possible
214 if (relatedFile.getName().endsWith(".info")) {
215 try {
216 String name = relatedFile.getName().replaceFirst("\\.info$",
217 "");
218 relatedFile.delete();
219 InfoCover.writeInfo(newDir, name, meta);
220 relatedFile.getParentFile().delete();
221 } catch (IOException e) {
222 Instance.getInstance().getTraceHandler().error(e);
223 }
224 } else {
225 relatedFile.renameTo(new File(newDir, relatedFile.getName()));
226 relatedFile.getParentFile().delete();
227 }
228 }
229
230 updateInfo(meta);
231 }
232
233 @Override
234 public Image getCustomSourceCover(String source) {
235 synchronized (lock) {
236 if (sourceCovers == null) {
237 sourceCovers = new HashMap<String, Image>();
238 }
239 }
240
241 synchronized (lock) {
242 Image img = sourceCovers.get(source);
243 if (img != null) {
244 return img;
245 }
246 }
247
248 File coverDir = getExpectedDir(source);
249 if (coverDir.isDirectory()) {
250 File cover = new File(coverDir, ".cover.png");
251 if (cover.exists()) {
252 InputStream in;
253 try {
254 in = new FileInputStream(cover);
255 try {
256 synchronized (lock) {
257 sourceCovers.put(source, new Image(in));
258 }
259 } finally {
260 in.close();
261 }
262 } catch (FileNotFoundException e) {
263 e.printStackTrace();
264 } catch (IOException e) {
265 Instance.getInstance().getTraceHandler()
266 .error(new IOException(
267 "Cannot load the existing custom source cover: "
268 + cover,
269 e));
270 }
271 }
272 }
273
274 synchronized (lock) {
275 return sourceCovers.get(source);
276 }
277 }
278
279 @Override
280 public Image getCustomAuthorCover(String author) {
281 synchronized (lock) {
282 if (authorCovers == null) {
283 authorCovers = new HashMap<String, Image>();
284 }
285 }
286
287 synchronized (lock) {
288 Image img = authorCovers.get(author);
289 if (img != null) {
290 return img;
291 }
292 }
293
294 File cover = getAuthorCoverFile(author);
295 if (cover.exists()) {
296 InputStream in;
297 try {
298 in = new FileInputStream(cover);
299 try {
300 synchronized (lock) {
301 authorCovers.put(author, new Image(in));
302 }
303 } finally {
304 in.close();
305 }
306 } catch (FileNotFoundException e) {
307 e.printStackTrace();
308 } catch (IOException e) {
309 Instance.getInstance().getTraceHandler()
310 .error(new IOException(
311 "Cannot load the existing custom author cover: "
312 + cover,
313 e));
314 }
315 }
316
317 synchronized (lock) {
318 return authorCovers.get(author);
319 }
320 }
321
322 @Override
323 public void setSourceCover(String source, String luid) throws IOException {
324 setSourceCover(source, getCover(luid));
325 }
326
327 @Override
328 public void setAuthorCover(String author, String luid) throws IOException {
329 setAuthorCover(author, getCover(luid));
330 }
331
332 /**
333 * Set the source cover to the given story cover.
334 *
335 * @param source
336 * the source to change
337 * @param coverImage
338 * the cover image
339 */
340 void setSourceCover(String source, Image coverImage) {
341 File dir = getExpectedDir(source);
342 dir.mkdirs();
343 File cover = new File(dir, ".cover");
344 try {
345 Instance.getInstance().getCache().saveAsImage(coverImage, cover,
346 true);
347 synchronized (lock) {
348 if (sourceCovers != null) {
349 sourceCovers.put(source, coverImage);
350 }
351 }
352 } catch (IOException e) {
353 Instance.getInstance().getTraceHandler().error(e);
354 }
355 }
356
357 /**
358 * Set the author cover to the given story cover.
359 *
360 * @param author
361 * the author to change
362 * @param coverImage
363 * the cover image
364 */
365 void setAuthorCover(String author, Image coverImage) {
366 File cover = getAuthorCoverFile(author);
367 cover.getParentFile().mkdirs();
368 try {
369 Instance.getInstance().getCache().saveAsImage(coverImage, cover,
370 true);
371 synchronized (lock) {
372 if (authorCovers != null) {
373 authorCovers.put(author, coverImage);
374 }
375 }
376 } catch (IOException e) {
377 Instance.getInstance().getTraceHandler().error(e);
378 }
379 }
380
381 @Override
382 public void imprt(BasicLibrary other, String luid, Progress pg)
383 throws IOException {
384 if (pg == null) {
385 pg = new Progress();
386 }
387
388 // Check if we can simply copy the files instead of the whole process
389 if (other instanceof LocalLibrary) {
390 LocalLibrary otherLocalLibrary = (LocalLibrary) other;
391
392 MetaData meta = otherLocalLibrary.getInfo(luid);
393 String expectedType = ""
394 + (meta != null && meta.isImageDocument() ? image : text);
395 if (meta != null && meta.getType().equals(expectedType)) {
396 File from = otherLocalLibrary.getExpectedDir(meta.getSource());
397 File to = this.getExpectedDir(meta.getSource());
398 List<File> relatedFiles = otherLocalLibrary
399 .getRelatedFiles(luid);
400 if (!relatedFiles.isEmpty()) {
401 pg.setMinMax(0, relatedFiles.size());
402 }
403
404 for (File relatedFile : relatedFiles) {
405 File target = new File(relatedFile.getAbsolutePath()
406 .replace(from.getAbsolutePath(),
407 to.getAbsolutePath()));
408 if (!relatedFile.equals(target)) {
409 target.getParentFile().mkdirs();
410 InputStream in = null;
411 try {
412 in = new FileInputStream(relatedFile);
413 IOUtils.write(in, target);
414 } catch (IOException e) {
415 if (in != null) {
416 try {
417 in.close();
418 } catch (Exception ee) {
419 }
420 }
421
422 pg.done();
423 throw e;
424 }
425 }
426
427 pg.add(1);
428 }
429
430 invalidateInfo();
431 pg.done();
432 return;
433 }
434 }
435
436 super.imprt(other, luid, pg);
437 }
438
439 /**
440 * Return the {@link OutputType} for this {@link Story}.
441 *
442 * @param meta
443 * the {@link Story} {@link MetaData}
444 *
445 * @return the type
446 */
447 private OutputType getOutputType(MetaData meta) {
448 if (meta != null && meta.isImageDocument()) {
449 return image;
450 }
451
452 return text;
453 }
454
455 /**
456 * Return the default {@link OutputType} for this kind of {@link Story}.
457 *
458 * @param imageDocument
459 * TRUE for images document, FALSE for text documents
460 *
461 * @return the type
462 */
463 public String getOutputType(boolean imageDocument) {
464 if (imageDocument) {
465 return image.toString();
466 }
467
468 return text.toString();
469 }
470
471 /**
472 * Get the target {@link File} related to the given <tt>.info</tt>
473 * {@link File} and {@link MetaData}.
474 *
475 * @param meta
476 * the meta
477 * @param infoFile
478 * the <tt>.info</tt> {@link File}
479 *
480 * @return the target {@link File}
481 */
482 private File getTargetFile(MetaData meta, File infoFile) {
483 // Replace .info with whatever is needed:
484 String path = infoFile.getPath();
485 path = path.substring(0, path.length() - ".info".length());
486 String newExt = getOutputType(meta).getDefaultExtension(true);
487
488 return new File(path + newExt);
489 }
490
491 /**
492 * The target (full path) where the {@link Story} related to this
493 * {@link MetaData} should be located on disk for a new {@link Story}.
494 *
495 * @param key
496 * the {@link Story} {@link MetaData}
497 *
498 * @return the target
499 */
500 private File getExpectedFile(MetaData key) {
501 String title = key.getTitle();
502 if (title == null) {
503 title = "";
504 }
505 title = title.replaceAll("[^a-zA-Z0-9._+-]", "_");
506 if (title.length() > 40) {
507 title = title.substring(0, 40);
508 }
509 return new File(getExpectedDir(key.getSource()),
510 key.getLuid() + "_" + title);
511 }
512
513 /**
514 * The directory (full path) where the new {@link Story} related to this
515 * {@link MetaData} should be located on disk.
516 *
517 * @param source
518 * the type (source)
519 *
520 * @return the target directory
521 */
522 private File getExpectedDir(String source) {
523 String sanitizedSource = source.replaceAll("[^a-zA-Z0-9._+/-]", "_");
524
525 while (sanitizedSource.startsWith("/")
526 || sanitizedSource.startsWith("_")) {
527 if (sanitizedSource.length() > 1) {
528 sanitizedSource = sanitizedSource.substring(1);
529 } else {
530 sanitizedSource = "";
531 }
532 }
533
534 sanitizedSource = sanitizedSource.replace("/", File.separator);
535
536 if (sanitizedSource.isEmpty()) {
537 sanitizedSource = "_EMPTY";
538 }
539
540 return new File(baseDir, sanitizedSource);
541 }
542
543 /**
544 * Return the full path to the file to use for the custom cover of this
545 * author.
546 * <p>
547 * One or more of the parent directories <b>MAY</b> not exist.
548 *
549 * @param author
550 * the author
551 *
552 * @return the custom cover file
553 */
554 private File getAuthorCoverFile(String author) {
555 File aDir = new File(baseDir, "_AUTHORS");
556 String hash = HashUtils.md5(author);
557 String ext = Instance.getInstance().getConfig()
558 .getString(Config.FILE_FORMAT_IMAGE_FORMAT_COVER);
559 return new File(aDir, hash + "." + ext.toLowerCase());
560 }
561
562 /**
563 * Return the list of files/directories on disk for this {@link Story}.
564 * <p>
565 * If the {@link Story} is not found, and empty list is returned.
566 *
567 * @param luid
568 * the {@link Story} LUID
569 *
570 * @return the list of {@link File}s
571 *
572 * @throws IOException
573 * if the {@link Story} was not found
574 */
575 private List<File> getRelatedFiles(String luid) throws IOException {
576 List<File> files = new ArrayList<File>();
577
578 MetaData meta = getInfo(luid);
579 if (meta == null) {
580 throw new IOException("Story not found: " + luid);
581 }
582
583 File infoFile = getStories(null).get(meta)[0];
584 File targetFile = getStories(null).get(meta)[1];
585
586 files.add(infoFile);
587 files.add(targetFile);
588
589 String readerExt = getOutputType(meta).getDefaultExtension(true);
590 String fileExt = getOutputType(meta).getDefaultExtension(false);
591
592 String path = targetFile.getAbsolutePath();
593 if (readerExt != null && !readerExt.equals(fileExt)) {
594 path = path.substring(0, path.length() - readerExt.length())
595 + fileExt;
596 File relatedFile = new File(path);
597
598 if (relatedFile.exists()) {
599 files.add(relatedFile);
600 }
601 }
602
603 String coverExt = "." + Instance.getInstance().getConfig()
604 .getString(Config.FILE_FORMAT_IMAGE_FORMAT_COVER).toLowerCase();
605 File coverFile = new File(path + coverExt);
606 if (!coverFile.exists()) {
607 coverFile = new File(
608 path.substring(0, path.length() - fileExt.length())
609 + coverExt);
610 }
611
612 if (coverFile.exists()) {
613 files.add(coverFile);
614 }
615
616 return files;
617 }
618
619 /**
620 * Fill the list of stories by reading the content of the local directory
621 * {@link LocalLibrary#baseDir}.
622 * <p>
623 * Will use a cached list when possible (see
624 * {@link BasicLibrary#invalidateInfo()}).
625 *
626 * @param pg
627 * the optional {@link Progress}
628 *
629 * @return the list of stories (for each item, the first {@link File} is the
630 * info file, the second file is the target {@link File})
631 */
632 private Map<MetaData, File[]> getStories(Progress pg) {
633 if (pg == null) {
634 pg = new Progress();
635 } else {
636 pg.setMinMax(0, 100);
637 }
638
639 Map<MetaData, File[]> stories = this.stories;
640 if (stories == null) {
641 stories = getStoriesDo(pg);
642 synchronized (lock) {
643 if (this.stories == null)
644 this.stories = stories;
645 else
646 stories = this.stories;
647 }
648 }
649
650 pg.done();
651 return stories;
652
653 }
654
655 /**
656 * Actually do the work of {@link LocalLibrary#getStories(Progress)} (i.e.,
657 * do not retrieve the cache).
658 *
659 * @param pg
660 * the optional {@link Progress}
661 *
662 * @return the list of stories (for each item, the first {@link File} is the
663 * info file, the second file is the target {@link File})
664 */
665 private synchronized Map<MetaData, File[]> getStoriesDo(Progress pg) {
666 if (pg == null) {
667 pg = new Progress();
668 } else {
669 pg.setMinMax(0, 100);
670 }
671
672 Map<MetaData, File[]> stories = new HashMap<MetaData, File[]>();
673
674 File[] dirs = baseDir.listFiles(new FileFilter() {
675 @Override
676 public boolean accept(File file) {
677 return file != null && file.isDirectory();
678 }
679 });
680
681 if (dirs != null) {
682 Progress pgDirs = new Progress(0, 100 * dirs.length);
683 pg.addProgress(pgDirs, 100);
684
685 for (File dir : dirs) {
686 Progress pgFiles = new Progress();
687 pgDirs.addProgress(pgFiles, 100);
688 pgDirs.setName("Loading from: " + dir.getName());
689
690 addToStories(stories, pgFiles, dir);
691
692 pgFiles.setName(null);
693 }
694
695 pgDirs.setName("Loading directories");
696 }
697
698 pg.done();
699
700 return stories;
701 }
702
703 private void addToStories(Map<MetaData, File[]> stories, Progress pgFiles,
704 File dir) {
705 File[] infoFilesAndSubdirs = dir.listFiles(new FileFilter() {
706 @Override
707 public boolean accept(File file) {
708 boolean info = file != null && file.isFile()
709 && file.getPath().toLowerCase().endsWith(".info");
710 boolean dir = file != null && file.isDirectory();
711 boolean isExpandedHtml = new File(file, "index.html").isFile();
712 return info || (dir && !isExpandedHtml);
713 }
714 });
715
716 if (pgFiles != null) {
717 pgFiles.setMinMax(0, infoFilesAndSubdirs.length);
718 }
719
720 for (File infoFileOrSubdir : infoFilesAndSubdirs) {
721 if (infoFileOrSubdir.isDirectory()) {
722 addToStories(stories, null, infoFileOrSubdir);
723 } else {
724 try {
725 MetaData meta = InfoReader.readMeta(infoFileOrSubdir,
726 false);
727 try {
728 int id = Integer.parseInt(meta.getLuid());
729 if (id > lastId) {
730 lastId = id;
731 }
732
733 stories.put(meta, new File[] { infoFileOrSubdir,
734 getTargetFile(meta, infoFileOrSubdir) });
735 } catch (Exception e) {
736 // not normal!!
737 throw new IOException("Cannot understand the LUID of "
738 + infoFileOrSubdir + ": " + meta.getLuid(), e);
739 }
740 } catch (IOException e) {
741 // We should not have not-supported files in the
742 // library
743 Instance.getInstance().getTraceHandler().error(
744 new IOException("Cannot load file from library: "
745 + infoFileOrSubdir, e));
746 }
747 }
748
749 if (pgFiles != null) {
750 pgFiles.add(1);
751 }
752 }
753 }
754}