test: improve flag files
[nikiroo-utils.git] / src / be / nikiroo / fanfix / library / LocalLibrary.java
CommitLineData
e42573a0 1package be.nikiroo.fanfix.library;
68e2c6d2 2
68e2c6d2
NR
3import java.io.File;
4import java.io.FileFilter;
14b57448 5import java.io.FileInputStream;
e1de8087 6import java.io.FileNotFoundException;
68e2c6d2 7import java.io.IOException;
14b57448 8import java.io.InputStream;
68e2c6d2
NR
9import java.util.ArrayList;
10import java.util.HashMap;
11import java.util.List;
12import java.util.Map;
13
e42573a0 14import be.nikiroo.fanfix.Instance;
68e2c6d2
NR
15import be.nikiroo.fanfix.bundles.Config;
16import be.nikiroo.fanfix.data.MetaData;
17import be.nikiroo.fanfix.data.Story;
18import be.nikiroo.fanfix.output.BasicOutput;
19import be.nikiroo.fanfix.output.BasicOutput.OutputType;
20import be.nikiroo.fanfix.output.InfoCover;
21import be.nikiroo.fanfix.supported.InfoReader;
22import be.nikiroo.utils.IOUtils;
16a81ef7 23import be.nikiroo.utils.Image;
68e2c6d2 24import be.nikiroo.utils.Progress;
3989dfc5 25import be.nikiroo.utils.StringUtils;
68e2c6d2
NR
26
27/**
28 * This {@link BasicLibrary} will store the stories locally on disk.
29 *
30 * @author niki
31 */
32public class LocalLibrary extends BasicLibrary {
33 private int lastId;
34 private Map<MetaData, File[]> stories; // Files: [ infoFile, TargetFile ]
16a81ef7 35 private Map<String, Image> sourceCovers;
3989dfc5 36 private Map<String, Image> authorCovers;
68e2c6d2
NR
37
38 private File baseDir;
39 private OutputType text;
40 private OutputType image;
41
e604986c
NR
42 /**
43 * Create a new {@link LocalLibrary} with the given back-end directory.
44 *
45 * @param baseDir
46 * the directory where to find the {@link Story} objects
47 */
48 public LocalLibrary(File baseDir) {
49 this(baseDir, Instance.getConfig().getString(
13fdb89a
NR
50 Config.FILE_FORMAT_NON_IMAGES_DOCUMENT_TYPE), Instance.getConfig()
51 .getString(Config.FILE_FORMAT_IMAGES_DOCUMENT_TYPE), false);
e604986c
NR
52 }
53
54 /**
55 * Create a new {@link LocalLibrary} with the given back-end directory.
56 *
57 * @param baseDir
58 * the directory where to find the {@link Story} objects
ff05b828
NR
59 * @param text
60 * the {@link OutputType} to use for non-image documents
61 * @param image
62 * the {@link OutputType} to use for image documents
63 * @param defaultIsHtml
64 * if the given text or image is invalid, use HTML by default (if
65 * not, it will be INFO_TEXT/CBZ by default)
e604986c
NR
66 */
67 public LocalLibrary(File baseDir, String text, String image,
68 boolean defaultIsHtml) {
ff05b828 69 this(baseDir, OutputType.valueOfAllOkUC(text,
e604986c 70 defaultIsHtml ? OutputType.HTML : OutputType.INFO_TEXT),
ff05b828 71 OutputType.valueOfAllOkUC(image,
e604986c
NR
72 defaultIsHtml ? OutputType.HTML : OutputType.CBZ));
73 }
74
68e2c6d2
NR
75 /**
76 * Create a new {@link LocalLibrary} with the given back-end directory.
77 *
78 * @param baseDir
79 * the directory where to find the {@link Story} objects
80 * @param text
e604986c 81 * the {@link OutputType} to use for non-image documents
68e2c6d2 82 * @param image
e604986c 83 * the {@link OutputType} to use for image documents
68e2c6d2
NR
84 */
85 public LocalLibrary(File baseDir, OutputType text, OutputType image) {
86 this.baseDir = baseDir;
87 this.text = text;
88 this.image = image;
89
90 this.lastId = 0;
91 this.stories = null;
b56c9d60 92 this.sourceCovers = null;
68e2c6d2
NR
93
94 baseDir.mkdirs();
95 }
96
97 @Override
98 protected List<MetaData> getMetas(Progress pg) {
99 return new ArrayList<MetaData>(getStories(pg).keySet());
100 }
101
102 @Override
0bb51c9c 103 public File getFile(String luid, Progress pg) throws IOException {
9e2fad36
NR
104 Instance.getTraceHandler().trace(
105 this.getClass().getSimpleName() + ": get file for " + luid);
106
107 File file = null;
108 String mess = "no file found for ";
109
110 MetaData meta = getInfo(luid);
9e2fad36 111 File[] files = getStories(pg).get(meta);
68e2c6d2 112 if (files != null) {
9e2fad36
NR
113 mess = "file retrieved for ";
114 file = files[1];
68e2c6d2
NR
115 }
116
9e2fad36 117 Instance.getTraceHandler().trace(
9b863b20
NR
118 this.getClass().getSimpleName() + ": " + mess + luid + " ("
119 + meta.getTitle() + ")");
9e2fad36
NR
120
121 return file;
68e2c6d2
NR
122 }
123
124 @Override
0bb51c9c 125 public Image getCover(String luid) throws IOException {
68e2c6d2
NR
126 MetaData meta = getInfo(luid);
127 if (meta != null) {
3ab45a6e
NR
128 if (meta.getCover() != null) {
129 return meta.getCover();
130 }
131
68e2c6d2
NR
132 File[] files = getStories(null).get(meta);
133 if (files != null) {
134 File infoFile = files[0];
135
136 try {
137 meta = InfoReader.readMeta(infoFile, true);
138 return meta.getCover();
139 } catch (IOException e) {
62c63b07 140 Instance.getTraceHandler().error(e);
68e2c6d2
NR
141 }
142 }
143 }
144
145 return null;
146 }
147
148 @Override
efa3c511 149 protected synchronized void updateInfo(MetaData meta) {
c8d48938 150 invalidateInfo();
efa3c511
NR
151 }
152
153 @Override
c8d48938 154 protected void invalidateInfo(String luid) {
68e2c6d2 155 stories = null;
b56c9d60 156 sourceCovers = null;
68e2c6d2
NR
157 }
158
159 @Override
160 protected synchronized int getNextId() {
14b57448 161 getStories(null); // make sure lastId is set
68e2c6d2
NR
162 return ++lastId;
163 }
164
165 @Override
166 protected void doDelete(String luid) throws IOException {
167 for (File file : getRelatedFiles(luid)) {
168 // TODO: throw an IOException if we cannot delete the files?
169 IOUtils.deltree(file);
e272f05f 170 file.getParentFile().delete();
68e2c6d2
NR
171 }
172 }
173
174 @Override
175 protected Story doSave(Story story, Progress pg) throws IOException {
176 MetaData meta = story.getMeta();
177
178 File expectedTarget = getExpectedFile(meta);
179 expectedTarget.getParentFile().mkdirs();
180
925298fd 181 BasicOutput it = BasicOutput.getOutput(getOutputType(meta), true, true);
68e2c6d2
NR
182 it.process(story, expectedTarget.getPath(), pg);
183
184 return story;
185 }
186
187 @Override
188 protected synchronized void saveMeta(MetaData meta, Progress pg)
189 throws IOException {
190 File newDir = getExpectedDir(meta.getSource());
191 if (!newDir.exists()) {
3a0605e6 192 newDir.mkdirs();
68e2c6d2
NR
193 }
194
195 List<File> relatedFiles = getRelatedFiles(meta.getLuid());
196 for (File relatedFile : relatedFiles) {
197 // TODO: this is not safe at all.
198 // We should copy all the files THEN delete them
199 // Maybe also adding some rollback cleanup if possible
200 if (relatedFile.getName().endsWith(".info")) {
201 try {
202 String name = relatedFile.getName().replaceFirst(
203 "\\.info$", "");
68e2c6d2 204 relatedFile.delete();
c8d48938 205 InfoCover.writeInfo(newDir, name, meta);
e272f05f 206 relatedFile.getParentFile().delete();
68e2c6d2 207 } catch (IOException e) {
62c63b07 208 Instance.getTraceHandler().error(e);
68e2c6d2
NR
209 }
210 } else {
211 relatedFile.renameTo(new File(newDir, relatedFile.getName()));
e272f05f 212 relatedFile.getParentFile().delete();
68e2c6d2
NR
213 }
214 }
215
c8d48938 216 invalidateInfo();
68e2c6d2
NR
217 }
218
14b57448 219 @Override
e1de8087 220 public synchronized Image getCustomSourceCover(String source) {
b56c9d60 221 if (sourceCovers == null) {
e1de8087
NR
222 sourceCovers = new HashMap<String, Image>();
223 }
224
225 Image img = sourceCovers.get(source);
226 if (img != null) {
227 return img;
b56c9d60
NR
228 }
229
3a0605e6 230 File coverDir = getExpectedDir(source);
e1de8087
NR
231 if (coverDir.isDirectory()) {
232 File cover = new File(coverDir, ".cover.png");
a09ef2bb
NR
233 if (cover.exists()) {
234 InputStream in;
e1de8087 235 try {
a09ef2bb
NR
236 in = new FileInputStream(cover);
237 try {
238 sourceCovers.put(source, new Image(in));
239 } finally {
240 in.close();
241 }
242 } catch (FileNotFoundException e) {
243 e.printStackTrace();
244 } catch (IOException e) {
245 Instance.getTraceHandler().error(
246 new IOException(
247 "Cannot load the existing custom source cover: "
248 + cover, e));
e1de8087 249 }
e1de8087 250 }
14b57448
NR
251 }
252
253 return sourceCovers.get(source);
254 }
255
3989dfc5
NR
256 @Override
257 public synchronized Image getCustomAuthorCover(String author) {
c956ff52
NR
258 if (authorCovers == null) {
259 authorCovers = new HashMap<String, Image>();
260 }
261
262 Image img = authorCovers.get(author);
263 if (img != null) {
264 return img;
265 }
266
3989dfc5
NR
267 File cover = getAuthorCoverFile(author);
268 if (cover.exists()) {
269 InputStream in;
270 try {
271 in = new FileInputStream(cover);
272 try {
273 authorCovers.put(author, new Image(in));
274 } finally {
275 in.close();
276 }
277 } catch (FileNotFoundException e) {
278 e.printStackTrace();
279 } catch (IOException e) {
280 Instance.getTraceHandler().error(
281 new IOException(
282 "Cannot load the existing custom author cover: "
283 + cover, e));
284 }
285 }
286
287 return authorCovers.get(author);
288 }
289
14b57448 290 @Override
0bb51c9c 291 public void setSourceCover(String source, String luid) throws IOException {
e1de8087
NR
292 setSourceCover(source, getCover(luid));
293 }
b56c9d60 294
3989dfc5 295 @Override
0bb51c9c 296 public void setAuthorCover(String author, String luid) throws IOException {
3989dfc5
NR
297 setAuthorCover(author, getCover(luid));
298 }
299
e1de8087 300 /**
3989dfc5 301 * Set the source cover to the given story cover.
e1de8087
NR
302 *
303 * @param source
304 * the source to change
305 * @param coverImage
306 * the cover image
307 */
308 synchronized void setSourceCover(String source, Image coverImage) {
cf45a4c4
NR
309 File dir = getExpectedDir(source);
310 dir.mkdirs();
311 File cover = new File(dir, ".cover");
14b57448 312 try {
e1de8087
NR
313 Instance.getCache().saveAsImage(coverImage, cover, true);
314 if (sourceCovers != null) {
315 sourceCovers.put(source, coverImage);
316 }
14b57448 317 } catch (IOException e) {
62c63b07 318 Instance.getTraceHandler().error(e);
14b57448
NR
319 }
320 }
321
3989dfc5
NR
322 /**
323 * Set the author cover to the given story cover.
324 *
325 * @param author
326 * the author to change
327 * @param coverImage
328 * the cover image
329 */
330 synchronized void setAuthorCover(String author, Image coverImage) {
331 File cover = getAuthorCoverFile(author);
332 cover.getParentFile().mkdirs();
333 try {
334 Instance.getCache().saveAsImage(coverImage, cover, true);
335 if (authorCovers != null) {
336 authorCovers.put(author, coverImage);
337 }
338 } catch (IOException e) {
339 Instance.getTraceHandler().error(e);
340 }
341 }
342
b89dfb6e
NR
343 @Override
344 public void imprt(BasicLibrary other, String luid, Progress pg)
345 throws IOException {
346 if (pg == null) {
347 pg = new Progress();
348 }
349
ff05b828 350 // Check if we can simply copy the files instead of the whole process
b89dfb6e 351 if (other instanceof LocalLibrary) {
ff05b828 352 LocalLibrary otherLocalLibrary = (LocalLibrary) other;
e604986c 353
e604986c
NR
354 MetaData meta = otherLocalLibrary.getInfo(luid);
355 String expectedType = ""
356 + (meta != null && meta.isImageDocument() ? image : text);
357 if (meta != null && meta.getType().equals(expectedType)) {
358 File from = otherLocalLibrary.getExpectedDir(meta.getSource());
b89dfb6e 359 File to = this.getExpectedDir(meta.getSource());
3a0605e6
NR
360 List<File> relatedFiles = otherLocalLibrary
361 .getRelatedFiles(luid);
362 if (!relatedFiles.isEmpty()) {
363 pg.setMinMax(0, relatedFiles.size());
b89dfb6e
NR
364 }
365
3a0605e6
NR
366 for (File relatedFile : relatedFiles) {
367 File target = new File(relatedFile.getAbsolutePath()
368 .replace(from.getAbsolutePath(),
369 to.getAbsolutePath()));
370 if (!relatedFile.equals(target)) {
e604986c 371 target.getParentFile().mkdirs();
b89dfb6e
NR
372 InputStream in = null;
373 try {
3a0605e6 374 in = new FileInputStream(relatedFile);
b89dfb6e
NR
375 IOUtils.write(in, target);
376 } catch (IOException e) {
377 if (in != null) {
378 try {
379 in.close();
380 } catch (Exception ee) {
381 }
382 }
383
384 pg.done();
385 throw e;
386 }
387 }
388
389 pg.add(1);
390 }
391
c8d48938 392 invalidateInfo();
b89dfb6e
NR
393 pg.done();
394 return;
395 }
396 }
397
398 super.imprt(other, luid, pg);
399 }
400
68e2c6d2
NR
401 /**
402 * Return the {@link OutputType} for this {@link Story}.
403 *
404 * @param meta
405 * the {@link Story} {@link MetaData}
406 *
407 * @return the type
408 */
409 private OutputType getOutputType(MetaData meta) {
410 if (meta != null && meta.isImageDocument()) {
411 return image;
68e2c6d2 412 }
d4449e96 413
211f7ddb 414 return text;
68e2c6d2 415 }
d4449e96 416
585ae2b8
N
417 /**
418 * Return the default {@link OutputType} for this kind of {@link Story}.
419 *
420 * @param imageDocument
d4449e96 421 * TRUE for images document, FALSE for text documents
585ae2b8
N
422 *
423 * @return the type
424 */
425 public String getOutputType(boolean imageDocument) {
426 if (imageDocument) {
427 return image.toString();
428 }
429
430 return text.toString();
431 }
68e2c6d2
NR
432
433 /**
434 * Get the target {@link File} related to the given <tt>.info</tt>
435 * {@link File} and {@link MetaData}.
436 *
437 * @param meta
438 * the meta
439 * @param infoFile
440 * the <tt>.info</tt> {@link File}
441 *
442 * @return the target {@link File}
443 */
444 private File getTargetFile(MetaData meta, File infoFile) {
445 // Replace .info with whatever is needed:
446 String path = infoFile.getPath();
447 path = path.substring(0, path.length() - ".info".length());
448 String newExt = getOutputType(meta).getDefaultExtension(true);
449
450 return new File(path + newExt);
451 }
452
453 /**
454 * The target (full path) where the {@link Story} related to this
455 * {@link MetaData} should be located on disk for a new {@link Story}.
456 *
457 * @param key
458 * the {@link Story} {@link MetaData}
459 *
460 * @return the target
461 */
462 private File getExpectedFile(MetaData key) {
463 String title = key.getTitle();
464 if (title == null) {
465 title = "";
466 }
467 title = title.replaceAll("[^a-zA-Z0-9._+-]", "_");
3989dfc5
NR
468 if (title.length() > 40) {
469 title = title.substring(0, 40);
470 }
68e2c6d2
NR
471 return new File(getExpectedDir(key.getSource()), key.getLuid() + "_"
472 + title);
473 }
474
475 /**
476 * The directory (full path) where the new {@link Story} related to this
477 * {@link MetaData} should be located on disk.
478 *
085a2f9a 479 * @param source
68e2c6d2
NR
480 * the type (source)
481 *
482 * @return the target directory
483 */
085a2f9a 484 private File getExpectedDir(String source) {
3a0605e6
NR
485 String sanitizedSource = source.replaceAll("[^a-zA-Z0-9._+/-]", "_");
486
3989dfc5
NR
487 while (sanitizedSource.startsWith("/")
488 || sanitizedSource.startsWith("_")) {
3a0605e6
NR
489 if (sanitizedSource.length() > 1) {
490 sanitizedSource = sanitizedSource.substring(1);
491 } else {
492 sanitizedSource = "";
493 }
494 }
495
496 sanitizedSource = sanitizedSource.replace("/", File.separator);
497
498 if (sanitizedSource.isEmpty()) {
3989dfc5 499 sanitizedSource = "_EMPTY";
3a0605e6
NR
500 }
501
085a2f9a 502 return new File(baseDir, sanitizedSource);
68e2c6d2
NR
503 }
504
3989dfc5
NR
505 /**
506 * Return the full path to the file to use for the custom cover of this
507 * author.
508 * <p>
509 * One or more of the parent directories <b>MAY</b> not exist.
510 *
511 * @param author
512 * the author
513 *
514 * @return the custom cover file
515 */
516 private File getAuthorCoverFile(String author) {
517 File aDir = new File(baseDir, "_AUTHORS");
518 String hash = StringUtils.getMd5Hash(author);
13fdb89a 519 String ext = Instance.getConfig().getString(Config.FILE_FORMAT_IMAGE_FORMAT_COVER);
3989dfc5
NR
520 return new File(aDir, hash + "." + ext.toLowerCase());
521 }
522
68e2c6d2
NR
523 /**
524 * Return the list of files/directories on disk for this {@link Story}.
525 * <p>
526 * If the {@link Story} is not found, and empty list is returned.
527 *
528 * @param luid
529 * the {@link Story} LUID
530 *
531 * @return the list of {@link File}s
532 *
533 * @throws IOException
534 * if the {@link Story} was not found
535 */
536 private List<File> getRelatedFiles(String luid) throws IOException {
537 List<File> files = new ArrayList<File>();
538
539 MetaData meta = getInfo(luid);
540 if (meta == null) {
541 throw new IOException("Story not found: " + luid);
211f7ddb 542 }
68e2c6d2 543
211f7ddb
NR
544 File infoFile = getStories(null).get(meta)[0];
545 File targetFile = getStories(null).get(meta)[1];
68e2c6d2 546
211f7ddb
NR
547 files.add(infoFile);
548 files.add(targetFile);
68e2c6d2 549
211f7ddb
NR
550 String readerExt = getOutputType(meta).getDefaultExtension(true);
551 String fileExt = getOutputType(meta).getDefaultExtension(false);
68e2c6d2 552
211f7ddb
NR
553 String path = targetFile.getAbsolutePath();
554 if (readerExt != null && !readerExt.equals(fileExt)) {
555 path = path.substring(0, path.length() - readerExt.length())
556 + fileExt;
557 File relatedFile = new File(path);
68e2c6d2 558
211f7ddb
NR
559 if (relatedFile.exists()) {
560 files.add(relatedFile);
68e2c6d2 561 }
211f7ddb 562 }
68e2c6d2 563
211f7ddb 564 String coverExt = "."
13fdb89a 565 + Instance.getConfig().getString(Config.FILE_FORMAT_IMAGE_FORMAT_COVER)
2a25f781 566 .toLowerCase();
211f7ddb
NR
567 File coverFile = new File(path + coverExt);
568 if (!coverFile.exists()) {
569 coverFile = new File(path.substring(0,
570 path.length() - fileExt.length())
571 + coverExt);
572 }
573
574 if (coverFile.exists()) {
575 files.add(coverFile);
68e2c6d2
NR
576 }
577
578 return files;
579 }
580
581 /**
582 * Fill the list of stories by reading the content of the local directory
583 * {@link LocalLibrary#baseDir}.
584 * <p>
585 * Will use a cached list when possible (see
c8d48938 586 * {@link BasicLibrary#invalidateInfo()}).
68e2c6d2
NR
587 *
588 * @param pg
589 * the optional {@link Progress}
590 *
9b863b20
NR
591 * @return the list of stories (for each item, the first {@link File} is the
592 * info file, the second file is the target {@link File})
68e2c6d2
NR
593 */
594 private synchronized Map<MetaData, File[]> getStories(Progress pg) {
595 if (pg == null) {
596 pg = new Progress();
597 } else {
598 pg.setMinMax(0, 100);
599 }
600
601 if (stories == null) {
602 stories = new HashMap<MetaData, File[]>();
603
604 lastId = 0;
605
606 File[] dirs = baseDir.listFiles(new FileFilter() {
211f7ddb 607 @Override
68e2c6d2
NR
608 public boolean accept(File file) {
609 return file != null && file.isDirectory();
610 }
611 });
612
b4f9071c
NR
613 if (dirs != null) {
614 Progress pgDirs = new Progress(0, 100 * dirs.length);
615 pg.addProgress(pgDirs, 100);
616
617 for (File dir : dirs) {
3a0605e6 618 Progress pgFiles = new Progress();
b4f9071c
NR
619 pgDirs.addProgress(pgFiles, 100);
620 pgDirs.setName("Loading from: " + dir.getName());
621
3a0605e6 622 addToStories(pgFiles, dir);
68e2c6d2 623
b4f9071c 624 pgFiles.setName(null);
14b57448
NR
625 }
626
b4f9071c 627 pgDirs.setName("Loading directories");
68e2c6d2 628 }
68e2c6d2
NR
629 }
630
b4f9071c 631 pg.done();
68e2c6d2
NR
632 return stories;
633 }
3a0605e6
NR
634
635 private void addToStories(Progress pgFiles, File dir) {
636 File[] infoFilesAndSubdirs = dir.listFiles(new FileFilter() {
637 @Override
638 public boolean accept(File file) {
639 boolean info = file != null && file.isFile()
640 && file.getPath().toLowerCase().endsWith(".info");
641 boolean dir = file != null && file.isDirectory();
efdbabcd
NR
642 boolean isExpandedHtml = new File(file, "index.html").isFile();
643 return info || (dir && !isExpandedHtml);
3a0605e6
NR
644 }
645 });
646
647 if (pgFiles != null) {
648 pgFiles.setMinMax(0, infoFilesAndSubdirs.length);
649 }
650
651 for (File infoFileOrSubdir : infoFilesAndSubdirs) {
652 if (pgFiles != null) {
653 pgFiles.setName(infoFileOrSubdir.getName());
654 }
655
656 if (infoFileOrSubdir.isDirectory()) {
657 addToStories(null, infoFileOrSubdir);
658 } else {
659 try {
660 MetaData meta = InfoReader
661 .readMeta(infoFileOrSubdir, false);
662 try {
663 int id = Integer.parseInt(meta.getLuid());
664 if (id > lastId) {
665 lastId = id;
666 }
667
668 stories.put(meta, new File[] { infoFileOrSubdir,
669 getTargetFile(meta, infoFileOrSubdir) });
670 } catch (Exception e) {
671 // not normal!!
672 throw new IOException("Cannot understand the LUID of "
673 + infoFileOrSubdir + ": " + meta.getLuid(), e);
674 }
675 } catch (IOException e) {
676 // We should not have not-supported files in the
677 // library
678 Instance.getTraceHandler().error(
679 new IOException("Cannot load file from library: "
680 + infoFileOrSubdir, e));
681 }
682 }
683
684 if (pgFiles != null) {
685 pgFiles.add(1);
686 }
687 }
688 }
68e2c6d2 689}