src/be/nikiroo/fanfix/library/LocalLibrary.java

   1 package be.nikiroo.fanfix.library;
   2
   3 import java.io.File;
   4 import java.io.FileFilter;
   5 import java.io.FileInputStream;
   6 import java.io.FileNotFoundException;
   7 import java.io.IOException;
   8 import java.io.InputStream;
   9 import java.util.ArrayList;
  10 import java.util.HashMap;
  11 import java.util.List;
  12 import java.util.Map;
  13
  14 import be.nikiroo.fanfix.Instance;
  15 import be.nikiroo.fanfix.bundles.Config;
  16 import be.nikiroo.fanfix.data.MetaData;
  17 import be.nikiroo.fanfix.data.Story;
  18 import be.nikiroo.fanfix.output.BasicOutput;
  19 import be.nikiroo.fanfix.output.BasicOutput.OutputType;
  20 import be.nikiroo.fanfix.output.InfoCover;
  21 import be.nikiroo.fanfix.supported.InfoReader;
  22 import be.nikiroo.utils.IOUtils;
  23 import be.nikiroo.utils.Image;
  24 import be.nikiroo.utils.Progress;
  25 import be.nikiroo.utils.StringUtils;
  26
  27 /**
  28  * This {@link BasicLibrary} will store the stories locally on disk.
  29  *
  30  * @author niki
  31  */
  32 public class LocalLibrary extends BasicLibrary {
  33         private int lastId;
  34         private Map<MetaData, File[]> stories; // Files: [ infoFile, TargetFile ]
  35         private Map<String, Image> sourceCovers;
  36         private Map<String, Image> authorCovers;
  37
  38         private File baseDir;
  39         private OutputType text;
  40         private OutputType image;
  41
  42         /**
  43          * Create a new {@link LocalLibrary} with the given back-end directory.
  44          *
  45          * @param baseDir
  46          *            the directory where to find the {@link Story} objects
  47          */
  48         public LocalLibrary(File baseDir) {
  49                 this(baseDir, Instance.getConfig().getString(
  50                                 Config.FILE_FORMAT_NON_IMAGES_DOCUMENT_TYPE), Instance.getConfig()
  51                                 .getString(Config.FILE_FORMAT_IMAGES_DOCUMENT_TYPE), false);
  52         }
  53
  54         /**
  55          * Create a new {@link LocalLibrary} with the given back-end directory.
  56          *
  57          * @param baseDir
  58          *            the directory where to find the {@link Story} objects
  59          * @param text
  60          *            the {@link OutputType} to use for non-image documents
  61          * @param image
  62          *            the {@link OutputType} to use for image documents
  63          * @param defaultIsHtml
  64          *            if the given text or image is invalid, use HTML by default (if
  65          *            not, it will be INFO_TEXT/CBZ by default)
  66          */
  67         public LocalLibrary(File baseDir, String text, String image,
  68                         boolean defaultIsHtml) {
  69                 this(baseDir, OutputType.valueOfAllOkUC(text,
  70                                 defaultIsHtml ? OutputType.HTML : OutputType.INFO_TEXT),
  71                                 OutputType.valueOfAllOkUC(image,
  72                                                 defaultIsHtml ? OutputType.HTML : OutputType.CBZ));
  73         }
  74
  75         /**
  76          * Create a new {@link LocalLibrary} with the given back-end directory.
  77          *
  78          * @param baseDir
  79          *            the directory where to find the {@link Story} objects
  80          * @param text
  81          *            the {@link OutputType} to use for non-image documents
  82          * @param image
  83          *            the {@link OutputType} to use for image documents
  84          */
  85         public LocalLibrary(File baseDir, OutputType text, OutputType image) {
  86                 this.baseDir = baseDir;
  87                 this.text = text;
  88                 this.image = image;
  89
  90                 this.lastId = 0;
  91                 this.stories = null;
  92                 this.sourceCovers = null;
  93
  94                 baseDir.mkdirs();
  95         }
  96
  97         @Override
  98         protected List<MetaData> getMetas(Progress pg) {
  99                 return new ArrayList<MetaData>(getStories(pg).keySet());
 100         }
 101
 102         @Override
 103         public File getFile(String luid, Progress pg) throws IOException {
 104                 Instance.getTraceHandler().trace(
 105                                 this.getClass().getSimpleName() + ": get file for " + luid);
 106
 107                 File file = null;
 108                 String mess = "no file found for ";
 109
 110                 MetaData meta = getInfo(luid);
 111                 File[] files = getStories(pg).get(meta);
 112                 if (files != null) {
 113                         mess = "file retrieved for ";
 114                         file = files[1];
 115                 }
 116
 117                 Instance.getTraceHandler().trace(
 118                                 this.getClass().getSimpleName() + ": " + mess + luid + " ("
 119                                                 + meta.getTitle() + ")");
 120
 121                 return file;
 122         }
 123
 124         @Override
 125         public Image getCover(String luid) throws IOException {
 126                 MetaData meta = getInfo(luid);
 127                 if (meta != null) {
 128                         if (meta.getCover() != null) {
 129                                 return meta.getCover();
 130                         }
 131
 132                         File[] files = getStories(null).get(meta);
 133                         if (files != null) {
 134                                 File infoFile = files[0];
 135
 136                                 try {
 137                                         meta = InfoReader.readMeta(infoFile, true);
 138                                         return meta.getCover();
 139                                 } catch (IOException e) {
 140                                         Instance.getTraceHandler().error(e);
 141                                 }
 142                         }
 143                 }
 144
 145                 return null;
 146         }
 147
 148         @Override
 149         protected synchronized void updateInfo(MetaData meta) {
 150                 invalidateInfo();
 151         }
 152
 153         @Override
 154         protected void invalidateInfo(String luid) {
 155                 stories = null;
 156                 sourceCovers = null;
 157         }
 158
 159         @Override
 160         protected synchronized int getNextId() {
 161                 getStories(null); // make sure lastId is set
 162                 return ++lastId;
 163         }
 164
 165         @Override
 166         protected void doDelete(String luid) throws IOException {
 167                 for (File file : getRelatedFiles(luid)) {
 168                         // TODO: throw an IOException if we cannot delete the files?
 169                         IOUtils.deltree(file);
 170                         file.getParentFile().delete();
 171                 }
 172         }
 173
 174         @Override
 175         protected Story doSave(Story story, Progress pg) throws IOException {
 176                 MetaData meta = story.getMeta();
 177
 178                 File expectedTarget = getExpectedFile(meta);
 179                 expectedTarget.getParentFile().mkdirs();
 180
 181                 BasicOutput it = BasicOutput.getOutput(getOutputType(meta), true, true);
 182                 it.process(story, expectedTarget.getPath(), pg);
 183
 184                 return story;
 185         }
 186
 187         @Override
 188         protected synchronized void saveMeta(MetaData meta, Progress pg)
 189                         throws IOException {
 190                 File newDir = getExpectedDir(meta.getSource());
 191                 if (!newDir.exists()) {
 192                         newDir.mkdirs();
 193                 }
 194
 195                 List<File> relatedFiles = getRelatedFiles(meta.getLuid());
 196                 for (File relatedFile : relatedFiles) {
 197                         // TODO: this is not safe at all.
 198                         // We should copy all the files THEN delete them
 199                         // Maybe also adding some rollback cleanup if possible
 200                         if (relatedFile.getName().endsWith(".info")) {
 201                                 try {
 202                                         String name = relatedFile.getName().replaceFirst(
 203                                                         "\\.info$", "");
 204                                         relatedFile.delete();
 205                                         InfoCover.writeInfo(newDir, name, meta);
 206                                         relatedFile.getParentFile().delete();
 207                                 } catch (IOException e) {
 208                                         Instance.getTraceHandler().error(e);
 209                                 }
 210                         } else {
 211                                 relatedFile.renameTo(new File(newDir, relatedFile.getName()));
 212                                 relatedFile.getParentFile().delete();
 213                         }
 214                 }
 215
 216                 invalidateInfo();
 217         }
 218
 219         @Override
 220         public synchronized Image getCustomSourceCover(String source) {
 221                 if (sourceCovers == null) {
 222                         sourceCovers = new HashMap<String, Image>();
 223                 }
 224
 225                 Image img = sourceCovers.get(source);
 226                 if (img != null) {
 227                         return img;
 228                 }
 229
 230                 File coverDir = getExpectedDir(source);
 231                 if (coverDir.isDirectory()) {
 232                         File cover = new File(coverDir, ".cover.png");
 233                         if (cover.exists()) {
 234                                 InputStream in;
 235                                 try {
 236                                         in = new FileInputStream(cover);
 237                                         try {
 238                                                 sourceCovers.put(source, new Image(in));
 239                                         } finally {
 240                                                 in.close();
 241                                         }
 242                                 } catch (FileNotFoundException e) {
 243                                         e.printStackTrace();
 244                                 } catch (IOException e) {
 245                                         Instance.getTraceHandler().error(
 246                                                         new IOException(
 247                                                                         "Cannot load the existing custom source cover: "
 248                                                                                         + cover, e));
 249                                 }
 250                         }
 251                 }
 252
 253                 return sourceCovers.get(source);
 254         }
 255
 256         @Override
 257         public synchronized Image getCustomAuthorCover(String author) {
 258                 if (authorCovers == null) {
 259                         authorCovers = new HashMap<String, Image>();
 260                 }
 261
 262                 Image img = authorCovers.get(author);
 263                 if (img != null) {
 264                         return img;
 265                 }
 266
 267                 File cover = getAuthorCoverFile(author);
 268                 if (cover.exists()) {
 269                         InputStream in;
 270                         try {
 271                                 in = new FileInputStream(cover);
 272                                 try {
 273                                         authorCovers.put(author, new Image(in));
 274                                 } finally {
 275                                         in.close();
 276                                 }
 277                         } catch (FileNotFoundException e) {
 278                                 e.printStackTrace();
 279                         } catch (IOException e) {
 280                                 Instance.getTraceHandler().error(
 281                                                 new IOException(
 282                                                                 "Cannot load the existing custom author cover: "
 283                                                                                 + cover, e));
 284                         }
 285                 }
 286
 287                 return authorCovers.get(author);
 288         }
 289
 290         @Override
 291         public void setSourceCover(String source, String luid) throws IOException {
 292                 setSourceCover(source, getCover(luid));
 293         }
 294
 295         @Override
 296         public void setAuthorCover(String author, String luid) throws IOException {
 297                 setAuthorCover(author, getCover(luid));
 298         }
 299
 300         /**
 301          * Set the source cover to the given story cover.
 302          *
 303          * @param source
 304          *            the source to change
 305          * @param coverImage
 306          *            the cover image
 307          */
 308         synchronized void setSourceCover(String source, Image coverImage) {
 309                 File dir = getExpectedDir(source);
 310                 dir.mkdirs();
 311                 File cover = new File(dir, ".cover");
 312                 try {
 313                         Instance.getCache().saveAsImage(coverImage, cover, true);
 314                         if (sourceCovers != null) {
 315                                 sourceCovers.put(source, coverImage);
 316                         }
 317                 } catch (IOException e) {
 318                         Instance.getTraceHandler().error(e);
 319                 }
 320         }
 321
 322         /**
 323          * Set the author cover to the given story cover.
 324          *
 325          * @param author
 326          *            the author to change
 327          * @param coverImage
 328          *            the cover image
 329          */
 330         synchronized void setAuthorCover(String author, Image coverImage) {
 331                 File cover = getAuthorCoverFile(author);
 332                 cover.getParentFile().mkdirs();
 333                 try {
 334                         Instance.getCache().saveAsImage(coverImage, cover, true);
 335                         if (authorCovers != null) {
 336                                 authorCovers.put(author, coverImage);
 337                         }
 338                 } catch (IOException e) {
 339                         Instance.getTraceHandler().error(e);
 340                 }
 341         }
 342
 343         @Override
 344         public void imprt(BasicLibrary other, String luid, Progress pg)
 345                         throws IOException {
 346                 if (pg == null) {
 347                         pg = new Progress();
 348                 }
 349
 350                 // Check if we can simply copy the files instead of the whole process
 351                 if (other instanceof LocalLibrary) {
 352                         LocalLibrary otherLocalLibrary = (LocalLibrary) other;
 353
 354                         MetaData meta = otherLocalLibrary.getInfo(luid);
 355                         String expectedType = ""
 356                                         + (meta != null && meta.isImageDocument() ? image : text);
 357                         if (meta != null && meta.getType().equals(expectedType)) {
 358                                 File from = otherLocalLibrary.getExpectedDir(meta.getSource());
 359                                 File to = this.getExpectedDir(meta.getSource());
 360                                 List<File> relatedFiles = otherLocalLibrary
 361                                                 .getRelatedFiles(luid);
 362                                 if (!relatedFiles.isEmpty()) {
 363                                         pg.setMinMax(0, relatedFiles.size());
 364                                 }
 365
 366                                 for (File relatedFile : relatedFiles) {
 367                                         File target = new File(relatedFile.getAbsolutePath()
 368                                                         .replace(from.getAbsolutePath(),
 369                                                                         to.getAbsolutePath()));
 370                                         if (!relatedFile.equals(target)) {
 371                                                 target.getParentFile().mkdirs();
 372                                                 InputStream in = null;
 373                                                 try {
 374                                                         in = new FileInputStream(relatedFile);
 375                                                         IOUtils.write(in, target);
 376                                                 } catch (IOException e) {
 377                                                         if (in != null) {
 378                                                                 try {
 379                                                                         in.close();
 380                                                                 } catch (Exception ee) {
 381                                                                 }
 382                                                         }
 383
 384                                                         pg.done();
 385                                                         throw e;
 386                                                 }
 387                                         }
 388
 389                                         pg.add(1);
 390                                 }
 391
 392                                 invalidateInfo();
 393                                 pg.done();
 394                                 return;
 395                         }
 396                 }
 397
 398                 super.imprt(other, luid, pg);
 399         }
 400
 401         /**
 402          * Return the {@link OutputType} for this {@link Story}.
 403          *
 404          * @param meta
 405          *            the {@link Story} {@link MetaData}
 406          *
 407          * @return the type
 408          */
 409         private OutputType getOutputType(MetaData meta) {
 410                 if (meta != null && meta.isImageDocument()) {
 411                         return image;
 412                 }
 413
 414                 return text;
 415         }
 416
 417         /**
 418          * Return the default {@link OutputType} for this kind of {@link Story}.
 419          *
 420          * @param imageDocument
 421          *            TRUE for images document, FALSE for text documents
 422          *
 423          * @return the type
 424          */
 425         public String getOutputType(boolean imageDocument) {
 426                 if (imageDocument) {
 427                         return image.toString();
 428                 }
 429
 430                 return text.toString();
 431         }
 432
 433         /**
 434          * Get the target {@link File} related to the given <tt>.info</tt>
 435          * {@link File} and {@link MetaData}.
 436          *
 437          * @param meta
 438          *            the meta
 439          * @param infoFile
 440          *            the <tt>.info</tt> {@link File}
 441          *
 442          * @return the target {@link File}
 443          */
 444         private File getTargetFile(MetaData meta, File infoFile) {
 445                 // Replace .info with whatever is needed:
 446                 String path = infoFile.getPath();
 447                 path = path.substring(0, path.length() - ".info".length());
 448                 String newExt = getOutputType(meta).getDefaultExtension(true);
 449
 450                 return new File(path + newExt);
 451         }
 452
 453         /**
 454          * The target (full path) where the {@link Story} related to this
 455          * {@link MetaData} should be located on disk for a new {@link Story}.
 456          *
 457          * @param key
 458          *            the {@link Story} {@link MetaData}
 459          *
 460          * @return the target
 461          */
 462         private File getExpectedFile(MetaData key) {
 463                 String title = key.getTitle();
 464                 if (title == null) {
 465                         title = "";
 466                 }
 467                 title = title.replaceAll("[^a-zA-Z0-9._+-]", "_");
 468                 if (title.length() > 40) {
 469                         title = title.substring(0, 40);
 470                 }
 471                 return new File(getExpectedDir(key.getSource()), key.getLuid() + "_"
 472                                 + title);
 473         }
 474
 475         /**
 476          * The directory (full path) where the new {@link Story} related to this
 477          * {@link MetaData} should be located on disk.
 478          *
 479          * @param source
 480          *            the type (source)
 481          *
 482          * @return the target directory
 483          */
 484         private File getExpectedDir(String source) {
 485                 String sanitizedSource = source.replaceAll("[^a-zA-Z0-9._+/-]", "_");
 486
 487                 while (sanitizedSource.startsWith("/")
 488                                 || sanitizedSource.startsWith("_")) {
 489                         if (sanitizedSource.length() > 1) {
 490                                 sanitizedSource = sanitizedSource.substring(1);
 491                         } else {
 492                                 sanitizedSource = "";
 493                         }
 494                 }
 495
 496                 sanitizedSource = sanitizedSource.replace("/", File.separator);
 497
 498                 if (sanitizedSource.isEmpty()) {
 499                         sanitizedSource = "_EMPTY";
 500                 }
 501
 502                 return new File(baseDir, sanitizedSource);
 503         }
 504
 505         /**
 506          * Return the full path to the file to use for the custom cover of this
 507          * author.
 508          * <p>
 509          * One or more of the parent directories <b>MAY</b> not exist.
 510          *
 511          * @param author
 512          *            the author
 513          *
 514          * @return the custom cover file
 515          */
 516         private File getAuthorCoverFile(String author) {
 517                 File aDir = new File(baseDir, "_AUTHORS");
 518                 String hash = StringUtils.getMd5Hash(author);
 519                 String ext = Instance.getConfig().getString(Config.FILE_FORMAT_IMAGE_FORMAT_COVER);
 520                 return new File(aDir, hash + "." + ext.toLowerCase());
 521         }
 522
 523         /**
 524          * Return the list of files/directories on disk for this {@link Story}.
 525          * <p>
 526          * If the {@link Story} is not found, and empty list is returned.
 527          *
 528          * @param luid
 529          *            the {@link Story} LUID
 530          *
 531          * @return the list of {@link File}s
 532          *
 533          * @throws IOException
 534          *             if the {@link Story} was not found
 535          */
 536         private List<File> getRelatedFiles(String luid) throws IOException {
 537                 List<File> files = new ArrayList<File>();
 538
 539                 MetaData meta = getInfo(luid);
 540                 if (meta == null) {
 541                         throw new IOException("Story not found: " + luid);
 542                 }
 543
 544                 File infoFile = getStories(null).get(meta)[0];
 545                 File targetFile = getStories(null).get(meta)[1];
 546
 547                 files.add(infoFile);
 548                 files.add(targetFile);
 549
 550                 String readerExt = getOutputType(meta).getDefaultExtension(true);
 551                 String fileExt = getOutputType(meta).getDefaultExtension(false);
 552
 553                 String path = targetFile.getAbsolutePath();
 554                 if (readerExt != null && !readerExt.equals(fileExt)) {
 555                         path = path.substring(0, path.length() - readerExt.length())
 556                                         + fileExt;
 557                         File relatedFile = new File(path);
 558
 559                         if (relatedFile.exists()) {
 560                                 files.add(relatedFile);
 561                         }
 562                 }
 563
 564                 String coverExt = "."
 565                                 + Instance.getConfig().getString(Config.FILE_FORMAT_IMAGE_FORMAT_COVER)
 566                                                 .toLowerCase();
 567                 File coverFile = new File(path + coverExt);
 568                 if (!coverFile.exists()) {
 569                         coverFile = new File(path.substring(0,
 570                                         path.length() - fileExt.length())
 571                                         + coverExt);
 572                 }
 573
 574                 if (coverFile.exists()) {
 575                         files.add(coverFile);
 576                 }
 577
 578                 return files;
 579         }
 580
 581         /**
 582          * Fill the list of stories by reading the content of the local directory
 583          * {@link LocalLibrary#baseDir}.
 584          * <p>
 585          * Will use a cached list when possible (see
 586          * {@link BasicLibrary#invalidateInfo()}).
 587          *
 588          * @param pg
 589          *            the optional {@link Progress}
 590          *
 591          * @return the list of stories (for each item, the first {@link File} is the
 592          *         info file, the second file is the target {@link File})
 593          */
 594         private synchronized Map<MetaData, File[]> getStories(Progress pg) {
 595                 if (pg == null) {
 596                         pg = new Progress();
 597                 } else {
 598                         pg.setMinMax(0, 100);
 599                 }
 600
 601                 if (stories == null) {
 602                         stories = new HashMap<MetaData, File[]>();
 603
 604                         lastId = 0;
 605
 606                         File[] dirs = baseDir.listFiles(new FileFilter() {
 607                                 @Override
 608                                 public boolean accept(File file) {
 609                                         return file != null && file.isDirectory();
 610                                 }
 611                         });
 612
 613                         if (dirs != null) {
 614                                 Progress pgDirs = new Progress(0, 100 * dirs.length);
 615                                 pg.addProgress(pgDirs, 100);
 616
 617                                 for (File dir : dirs) {
 618                                         Progress pgFiles = new Progress();
 619                                         pgDirs.addProgress(pgFiles, 100);
 620                                         pgDirs.setName("Loading from: " + dir.getName());
 621
 622                                         addToStories(pgFiles, dir);
 623
 624                                         pgFiles.setName(null);
 625                                 }
 626
 627                                 pgDirs.setName("Loading directories");
 628                         }
 629                 }
 630
 631                 pg.done();
 632                 return stories;
 633         }
 634
 635         private void addToStories(Progress pgFiles, File dir) {
 636                 File[] infoFilesAndSubdirs = dir.listFiles(new FileFilter() {
 637                         @Override
 638                         public boolean accept(File file) {
 639                                 boolean info = file != null && file.isFile()
 640                                                 && file.getPath().toLowerCase().endsWith(".info");
 641                                 boolean dir = file != null && file.isDirectory();
 642                                 boolean isExpandedHtml = new File(file, "index.html").isFile();
 643                                 return info || (dir && !isExpandedHtml);
 644                         }
 645                 });
 646
 647                 if (pgFiles != null) {
 648                         pgFiles.setMinMax(0, infoFilesAndSubdirs.length);
 649                 }
 650
 651                 for (File infoFileOrSubdir : infoFilesAndSubdirs) {
 652                         if (pgFiles != null) {
 653                                 pgFiles.setName(infoFileOrSubdir.getName());
 654                         }
 655
 656                         if (infoFileOrSubdir.isDirectory()) {
 657                                 addToStories(null, infoFileOrSubdir);
 658                         } else {
 659                                 try {
 660                                         MetaData meta = InfoReader
 661                                                         .readMeta(infoFileOrSubdir, false);
 662                                         try {
 663                                                 int id = Integer.parseInt(meta.getLuid());
 664                                                 if (id > lastId) {
 665                                                         lastId = id;
 666                                                 }
 667
 668                                                 stories.put(meta, new File[] { infoFileOrSubdir,
 669                                                                 getTargetFile(meta, infoFileOrSubdir) });
 670                                         } catch (Exception e) {
 671                                                 // not normal!!
 672                                                 throw new IOException("Cannot understand the LUID of "
 673                                                                 + infoFileOrSubdir + ": " + meta.getLuid(), e);
 674                                         }
 675                                 } catch (IOException e) {
 676                                         // We should not have not-supported files in the
 677                                         // library
 678                                         Instance.getTraceHandler().error(
 679                                                         new IOException("Cannot load file from library: "
 680                                                                         + infoFileOrSubdir, e));
 681                                 }
 682                         }
 683
 684                         if (pgFiles != null) {
 685                                 pgFiles.add(1);
 686                         }
 687                 }
 688         }
 689 }