src/be/nikiroo/fanfix/library/LocalLibrary.java

   1 package be.nikiroo.fanfix.library;
   2
   3 import java.io.File;
   4 import java.io.FileFilter;
   5 import java.io.FileInputStream;
   6 import java.io.FileNotFoundException;
   7 import java.io.IOException;
   8 import java.io.InputStream;
   9 import java.util.ArrayList;
  10 import java.util.HashMap;
  11 import java.util.List;
  12 import java.util.Map;
  13
  14 import be.nikiroo.fanfix.Instance;
  15 import be.nikiroo.fanfix.bundles.Config;
  16 import be.nikiroo.fanfix.bundles.ConfigBundle;
  17 import be.nikiroo.fanfix.bundles.UiConfigBundle;
  18 import be.nikiroo.fanfix.data.MetaData;
  19 import be.nikiroo.fanfix.data.Story;
  20 import be.nikiroo.fanfix.output.BasicOutput;
  21 import be.nikiroo.fanfix.output.BasicOutput.OutputType;
  22 import be.nikiroo.fanfix.output.InfoCover;
  23 import be.nikiroo.fanfix.supported.InfoReader;
  24 import be.nikiroo.utils.IOUtils;
  25 import be.nikiroo.utils.Image;
  26 import be.nikiroo.utils.Progress;
  27 import be.nikiroo.utils.StringUtils;
  28
  29 /**
  30  * This {@link BasicLibrary} will store the stories locally on disk.
  31  *
  32  * @author niki
  33  */
  34 public class LocalLibrary extends BasicLibrary {
  35         private int lastId;
  36         private Map<MetaData, File[]> stories; // Files: [ infoFile, TargetFile ]
  37         private Map<String, Image> sourceCovers;
  38         private Map<String, Image> authorCovers;
  39
  40         private File baseDir;
  41         private OutputType text;
  42         private OutputType image;
  43
  44         /**
  45          * Create a new {@link LocalLibrary} with the given back-end directory.
  46          *
  47          * @param baseDir the directory where to find the {@link Story} objects
  48          * @param config  the configuration used to know which kind of default
  49          *                {@link OutputType} to use for images and non-images stories
  50          */
  51         public LocalLibrary(File baseDir, ConfigBundle config) {
  52                 this(baseDir, //
  53                                 config.getString(Config.FILE_FORMAT_NON_IMAGES_DOCUMENT_TYPE),
  54                                 config.getString(Config.FILE_FORMAT_IMAGES_DOCUMENT_TYPE), false);
  55         }
  56
  57         /**
  58          * Create a new {@link LocalLibrary} with the given back-end directory.
  59          *
  60          * @param baseDir
  61          *            the directory where to find the {@link Story} objects
  62          * @param text
  63          *            the {@link OutputType} to use for non-image documents
  64          * @param image
  65          *            the {@link OutputType} to use for image documents
  66          * @param defaultIsHtml
  67          *            if the given text or image is invalid, use HTML by default (if
  68          *            not, it will be INFO_TEXT/CBZ by default)
  69          */
  70         public LocalLibrary(File baseDir, String text, String image,
  71                         boolean defaultIsHtml) {
  72                 this(baseDir, OutputType.valueOfAllOkUC(text,
  73                                 defaultIsHtml ? OutputType.HTML : OutputType.INFO_TEXT),
  74                                 OutputType.valueOfAllOkUC(image,
  75                                                 defaultIsHtml ? OutputType.HTML : OutputType.CBZ));
  76         }
  77
  78         /**
  79          * Create a new {@link LocalLibrary} with the given back-end directory.
  80          *
  81          * @param baseDir
  82          *            the directory where to find the {@link Story} objects
  83          * @param text
  84          *            the {@link OutputType} to use for non-image documents
  85          * @param image
  86          *            the {@link OutputType} to use for image documents
  87          */
  88         public LocalLibrary(File baseDir, OutputType text, OutputType image) {
  89                 this.baseDir = baseDir;
  90                 this.text = text;
  91                 this.image = image;
  92
  93                 this.lastId = 0;
  94                 this.stories = null;
  95                 this.sourceCovers = null;
  96
  97                 baseDir.mkdirs();
  98         }
  99
 100         @Override
 101         protected List<MetaData> getMetas(Progress pg) {
 102                 return new ArrayList<MetaData>(getStories(pg).keySet());
 103         }
 104
 105         @Override
 106         public File getFile(String luid, Progress pg) throws IOException {
 107                 Instance.getInstance().getTraceHandler().trace(this.getClass().getSimpleName() + ": get file for " + luid);
 108
 109                 File file = null;
 110                 String mess = "no file found for ";
 111
 112                 MetaData meta = getInfo(luid);
 113                 File[] files = getStories(pg).get(meta);
 114                 if (files != null) {
 115                         mess = "file retrieved for ";
 116                         file = files[1];
 117                 }
 118
 119                 Instance.getInstance().getTraceHandler()
 120                                 .trace(this.getClass().getSimpleName() + ": " + mess + luid + " (" + meta.getTitle() + ")");
 121
 122                 return file;
 123         }
 124
 125         @Override
 126         public Image getCover(String luid) throws IOException {
 127                 MetaData meta = getInfo(luid);
 128                 if (meta != null) {
 129                         if (meta.getCover() != null) {
 130                                 return meta.getCover();
 131                         }
 132
 133                         File[] files = getStories(null).get(meta);
 134                         if (files != null) {
 135                                 File infoFile = files[0];
 136
 137                                 try {
 138                                         meta = InfoReader.readMeta(infoFile, true);
 139                                         return meta.getCover();
 140                                 } catch (IOException e) {
 141                                         Instance.getInstance().getTraceHandler().error(e);
 142                                 }
 143                         }
 144                 }
 145
 146                 return null;
 147         }
 148
 149         @Override
 150         protected synchronized void updateInfo(MetaData meta) {
 151                 invalidateInfo();
 152         }
 153
 154         @Override
 155         protected void invalidateInfo(String luid) {
 156                 stories = null;
 157                 sourceCovers = null;
 158         }
 159
 160         @Override
 161         protected synchronized int getNextId() {
 162                 getStories(null); // make sure lastId is set
 163                 return ++lastId;
 164         }
 165
 166         @Override
 167         protected void doDelete(String luid) throws IOException {
 168                 for (File file : getRelatedFiles(luid)) {
 169                         // TODO: throw an IOException if we cannot delete the files?
 170                         IOUtils.deltree(file);
 171                         file.getParentFile().delete();
 172                 }
 173         }
 174
 175         @Override
 176         protected Story doSave(Story story, Progress pg) throws IOException {
 177                 MetaData meta = story.getMeta();
 178
 179                 File expectedTarget = getExpectedFile(meta);
 180                 expectedTarget.getParentFile().mkdirs();
 181
 182                 BasicOutput it = BasicOutput.getOutput(getOutputType(meta), true, true);
 183                 it.process(story, expectedTarget.getPath(), pg);
 184
 185                 return story;
 186         }
 187
 188         @Override
 189         protected synchronized void saveMeta(MetaData meta, Progress pg)
 190                         throws IOException {
 191                 File newDir = getExpectedDir(meta.getSource());
 192                 if (!newDir.exists()) {
 193                         newDir.mkdirs();
 194                 }
 195
 196                 List<File> relatedFiles = getRelatedFiles(meta.getLuid());
 197                 for (File relatedFile : relatedFiles) {
 198                         // TODO: this is not safe at all.
 199                         // We should copy all the files THEN delete them
 200                         // Maybe also adding some rollback cleanup if possible
 201                         if (relatedFile.getName().endsWith(".info")) {
 202                                 try {
 203                                         String name = relatedFile.getName().replaceFirst(
 204                                                         "\\.info$", "");
 205                                         relatedFile.delete();
 206                                         InfoCover.writeInfo(newDir, name, meta);
 207                                         relatedFile.getParentFile().delete();
 208                                 } catch (IOException e) {
 209                                         Instance.getInstance().getTraceHandler().error(e);
 210                                 }
 211                         } else {
 212                                 relatedFile.renameTo(new File(newDir, relatedFile.getName()));
 213                                 relatedFile.getParentFile().delete();
 214                         }
 215                 }
 216
 217                 invalidateInfo();
 218         }
 219
 220         @Override
 221         public synchronized Image getCustomSourceCover(String source) {
 222                 if (sourceCovers == null) {
 223                         sourceCovers = new HashMap<String, Image>();
 224                 }
 225
 226                 Image img = sourceCovers.get(source);
 227                 if (img != null) {
 228                         return img;
 229                 }
 230
 231                 File coverDir = getExpectedDir(source);
 232                 if (coverDir.isDirectory()) {
 233                         File cover = new File(coverDir, ".cover.png");
 234                         if (cover.exists()) {
 235                                 InputStream in;
 236                                 try {
 237                                         in = new FileInputStream(cover);
 238                                         try {
 239                                                 sourceCovers.put(source, new Image(in));
 240                                         } finally {
 241                                                 in.close();
 242                                         }
 243                                 } catch (FileNotFoundException e) {
 244                                         e.printStackTrace();
 245                                 } catch (IOException e) {
 246                                         Instance.getInstance().getTraceHandler()
 247                                                         .error(new IOException("Cannot load the existing custom source cover: " + cover, e));
 248                                 }
 249                         }
 250                 }
 251
 252                 return sourceCovers.get(source);
 253         }
 254
 255         @Override
 256         public synchronized Image getCustomAuthorCover(String author) {
 257                 if (authorCovers == null) {
 258                         authorCovers = new HashMap<String, Image>();
 259                 }
 260
 261                 Image img = authorCovers.get(author);
 262                 if (img != null) {
 263                         return img;
 264                 }
 265
 266                 File cover = getAuthorCoverFile(author);
 267                 if (cover.exists()) {
 268                         InputStream in;
 269                         try {
 270                                 in = new FileInputStream(cover);
 271                                 try {
 272                                         authorCovers.put(author, new Image(in));
 273                                 } finally {
 274                                         in.close();
 275                                 }
 276                         } catch (FileNotFoundException e) {
 277                                 e.printStackTrace();
 278                         } catch (IOException e) {
 279                                 Instance.getInstance().getTraceHandler()
 280                                                 .error(new IOException("Cannot load the existing custom author cover: " + cover, e));
 281                         }
 282                 }
 283
 284                 return authorCovers.get(author);
 285         }
 286
 287         @Override
 288         public void setSourceCover(String source, String luid) throws IOException {
 289                 setSourceCover(source, getCover(luid));
 290         }
 291
 292         @Override
 293         public void setAuthorCover(String author, String luid) throws IOException {
 294                 setAuthorCover(author, getCover(luid));
 295         }
 296
 297         /**
 298          * Set the source cover to the given story cover.
 299          *
 300          * @param source
 301          *            the source to change
 302          * @param coverImage
 303          *            the cover image
 304          */
 305         synchronized void setSourceCover(String source, Image coverImage) {
 306                 File dir = getExpectedDir(source);
 307                 dir.mkdirs();
 308                 File cover = new File(dir, ".cover");
 309                 try {
 310                         Instance.getInstance().getCache().saveAsImage(coverImage, cover, true);
 311                         if (sourceCovers != null) {
 312                                 sourceCovers.put(source, coverImage);
 313                         }
 314                 } catch (IOException e) {
 315                         Instance.getInstance().getTraceHandler().error(e);
 316                 }
 317         }
 318
 319         /**
 320          * Set the author cover to the given story cover.
 321          *
 322          * @param author
 323          *            the author to change
 324          * @param coverImage
 325          *            the cover image
 326          */
 327         synchronized void setAuthorCover(String author, Image coverImage) {
 328                 File cover = getAuthorCoverFile(author);
 329                 cover.getParentFile().mkdirs();
 330                 try {
 331                         Instance.getInstance().getCache().saveAsImage(coverImage, cover, true);
 332                         if (authorCovers != null) {
 333                                 authorCovers.put(author, coverImage);
 334                         }
 335                 } catch (IOException e) {
 336                         Instance.getInstance().getTraceHandler().error(e);
 337                 }
 338         }
 339
 340         @Override
 341         public void imprt(BasicLibrary other, String luid, Progress pg)
 342                         throws IOException {
 343                 if (pg == null) {
 344                         pg = new Progress();
 345                 }
 346
 347                 // Check if we can simply copy the files instead of the whole process
 348                 if (other instanceof LocalLibrary) {
 349                         LocalLibrary otherLocalLibrary = (LocalLibrary) other;
 350
 351                         MetaData meta = otherLocalLibrary.getInfo(luid);
 352                         String expectedType = ""
 353                                         + (meta != null && meta.isImageDocument() ? image : text);
 354                         if (meta != null && meta.getType().equals(expectedType)) {
 355                                 File from = otherLocalLibrary.getExpectedDir(meta.getSource());
 356                                 File to = this.getExpectedDir(meta.getSource());
 357                                 List<File> relatedFiles = otherLocalLibrary
 358                                                 .getRelatedFiles(luid);
 359                                 if (!relatedFiles.isEmpty()) {
 360                                         pg.setMinMax(0, relatedFiles.size());
 361                                 }
 362
 363                                 for (File relatedFile : relatedFiles) {
 364                                         File target = new File(relatedFile.getAbsolutePath()
 365                                                         .replace(from.getAbsolutePath(),
 366                                                                         to.getAbsolutePath()));
 367                                         if (!relatedFile.equals(target)) {
 368                                                 target.getParentFile().mkdirs();
 369                                                 InputStream in = null;
 370                                                 try {
 371                                                         in = new FileInputStream(relatedFile);
 372                                                         IOUtils.write(in, target);
 373                                                 } catch (IOException e) {
 374                                                         if (in != null) {
 375                                                                 try {
 376                                                                         in.close();
 377                                                                 } catch (Exception ee) {
 378                                                                 }
 379                                                         }
 380
 381                                                         pg.done();
 382                                                         throw e;
 383                                                 }
 384                                         }
 385
 386                                         pg.add(1);
 387                                 }
 388
 389                                 invalidateInfo();
 390                                 pg.done();
 391                                 return;
 392                         }
 393                 }
 394
 395                 super.imprt(other, luid, pg);
 396         }
 397
 398         /**
 399          * Return the {@link OutputType} for this {@link Story}.
 400          *
 401          * @param meta
 402          *            the {@link Story} {@link MetaData}
 403          *
 404          * @return the type
 405          */
 406         private OutputType getOutputType(MetaData meta) {
 407                 if (meta != null && meta.isImageDocument()) {
 408                         return image;
 409                 }
 410
 411                 return text;
 412         }
 413
 414         /**
 415          * Return the default {@link OutputType} for this kind of {@link Story}.
 416          *
 417          * @param imageDocument
 418          *            TRUE for images document, FALSE for text documents
 419          *
 420          * @return the type
 421          */
 422         public String getOutputType(boolean imageDocument) {
 423                 if (imageDocument) {
 424                         return image.toString();
 425                 }
 426
 427                 return text.toString();
 428         }
 429
 430         /**
 431          * Get the target {@link File} related to the given <tt>.info</tt>
 432          * {@link File} and {@link MetaData}.
 433          *
 434          * @param meta
 435          *            the meta
 436          * @param infoFile
 437          *            the <tt>.info</tt> {@link File}
 438          *
 439          * @return the target {@link File}
 440          */
 441         private File getTargetFile(MetaData meta, File infoFile) {
 442                 // Replace .info with whatever is needed:
 443                 String path = infoFile.getPath();
 444                 path = path.substring(0, path.length() - ".info".length());
 445                 String newExt = getOutputType(meta).getDefaultExtension(true);
 446
 447                 return new File(path + newExt);
 448         }
 449
 450         /**
 451          * The target (full path) where the {@link Story} related to this
 452          * {@link MetaData} should be located on disk for a new {@link Story}.
 453          *
 454          * @param key
 455          *            the {@link Story} {@link MetaData}
 456          *
 457          * @return the target
 458          */
 459         private File getExpectedFile(MetaData key) {
 460                 String title = key.getTitle();
 461                 if (title == null) {
 462                         title = "";
 463                 }
 464                 title = title.replaceAll("[^a-zA-Z0-9._+-]", "_");
 465                 if (title.length() > 40) {
 466                         title = title.substring(0, 40);
 467                 }
 468                 return new File(getExpectedDir(key.getSource()), key.getLuid() + "_"
 469                                 + title);
 470         }
 471
 472         /**
 473          * The directory (full path) where the new {@link Story} related to this
 474          * {@link MetaData} should be located on disk.
 475          *
 476          * @param source
 477          *            the type (source)
 478          *
 479          * @return the target directory
 480          */
 481         private File getExpectedDir(String source) {
 482                 String sanitizedSource = source.replaceAll("[^a-zA-Z0-9._+/-]", "_");
 483
 484                 while (sanitizedSource.startsWith("/")
 485                                 || sanitizedSource.startsWith("_")) {
 486                         if (sanitizedSource.length() > 1) {
 487                                 sanitizedSource = sanitizedSource.substring(1);
 488                         } else {
 489                                 sanitizedSource = "";
 490                         }
 491                 }
 492
 493                 sanitizedSource = sanitizedSource.replace("/", File.separator);
 494
 495                 if (sanitizedSource.isEmpty()) {
 496                         sanitizedSource = "_EMPTY";
 497                 }
 498
 499                 return new File(baseDir, sanitizedSource);
 500         }
 501
 502         /**
 503          * Return the full path to the file to use for the custom cover of this
 504          * author.
 505          * <p>
 506          * One or more of the parent directories <b>MAY</b> not exist.
 507          *
 508          * @param author
 509          *            the author
 510          *
 511          * @return the custom cover file
 512          */
 513         private File getAuthorCoverFile(String author) {
 514                 File aDir = new File(baseDir, "_AUTHORS");
 515                 String hash = StringUtils.getMd5Hash(author);
 516                 String ext = Instance.getInstance().getConfig().getString(Config.FILE_FORMAT_IMAGE_FORMAT_COVER);
 517                 return new File(aDir, hash + "." + ext.toLowerCase());
 518         }
 519
 520         /**
 521          * Return the list of files/directories on disk for this {@link Story}.
 522          * <p>
 523          * If the {@link Story} is not found, and empty list is returned.
 524          *
 525          * @param luid
 526          *            the {@link Story} LUID
 527          *
 528          * @return the list of {@link File}s
 529          *
 530          * @throws IOException
 531          *             if the {@link Story} was not found
 532          */
 533         private List<File> getRelatedFiles(String luid) throws IOException {
 534                 List<File> files = new ArrayList<File>();
 535
 536                 MetaData meta = getInfo(luid);
 537                 if (meta == null) {
 538                         throw new IOException("Story not found: " + luid);
 539                 }
 540
 541                 File infoFile = getStories(null).get(meta)[0];
 542                 File targetFile = getStories(null).get(meta)[1];
 543
 544                 files.add(infoFile);
 545                 files.add(targetFile);
 546
 547                 String readerExt = getOutputType(meta).getDefaultExtension(true);
 548                 String fileExt = getOutputType(meta).getDefaultExtension(false);
 549
 550                 String path = targetFile.getAbsolutePath();
 551                 if (readerExt != null && !readerExt.equals(fileExt)) {
 552                         path = path.substring(0, path.length() - readerExt.length())
 553                                         + fileExt;
 554                         File relatedFile = new File(path);
 555
 556                         if (relatedFile.exists()) {
 557                                 files.add(relatedFile);
 558                         }
 559                 }
 560
 561                 String coverExt = "."
 562                                 + Instance.getInstance().getConfig().getString(Config.FILE_FORMAT_IMAGE_FORMAT_COVER).toLowerCase();
 563                 File coverFile = new File(path + coverExt);
 564                 if (!coverFile.exists()) {
 565                         coverFile = new File(path.substring(0,
 566                                         path.length() - fileExt.length())
 567                                         + coverExt);
 568                 }
 569
 570                 if (coverFile.exists()) {
 571                         files.add(coverFile);
 572                 }
 573
 574                 return files;
 575         }
 576
 577         /**
 578          * Fill the list of stories by reading the content of the local directory
 579          * {@link LocalLibrary#baseDir}.
 580          * <p>
 581          * Will use a cached list when possible (see
 582          * {@link BasicLibrary#invalidateInfo()}).
 583          *
 584          * @param pg
 585          *            the optional {@link Progress}
 586          *
 587          * @return the list of stories (for each item, the first {@link File} is the
 588          *         info file, the second file is the target {@link File})
 589          */
 590         private synchronized Map<MetaData, File[]> getStories(Progress pg) {
 591                 if (pg == null) {
 592                         pg = new Progress();
 593                 } else {
 594                         pg.setMinMax(0, 100);
 595                 }
 596
 597                 if (stories == null) {
 598                         stories = new HashMap<MetaData, File[]>();
 599
 600                         lastId = 0;
 601
 602                         File[] dirs = baseDir.listFiles(new FileFilter() {
 603                                 @Override
 604                                 public boolean accept(File file) {
 605                                         return file != null && file.isDirectory();
 606                                 }
 607                         });
 608
 609                         if (dirs != null) {
 610                                 Progress pgDirs = new Progress(0, 100 * dirs.length);
 611                                 pg.addProgress(pgDirs, 100);
 612
 613                                 for (File dir : dirs) {
 614                                         Progress pgFiles = new Progress();
 615                                         pgDirs.addProgress(pgFiles, 100);
 616                                         pgDirs.setName("Loading from: " + dir.getName());
 617
 618                                         addToStories(pgFiles, dir);
 619
 620                                         pgFiles.setName(null);
 621                                 }
 622
 623                                 pgDirs.setName("Loading directories");
 624                         }
 625                 }
 626
 627                 pg.done();
 628                 return stories;
 629         }
 630
 631         private void addToStories(Progress pgFiles, File dir) {
 632                 File[] infoFilesAndSubdirs = dir.listFiles(new FileFilter() {
 633                         @Override
 634                         public boolean accept(File file) {
 635                                 boolean info = file != null && file.isFile()
 636                                                 && file.getPath().toLowerCase().endsWith(".info");
 637                                 boolean dir = file != null && file.isDirectory();
 638                                 boolean isExpandedHtml = new File(file, "index.html").isFile();
 639                                 return info || (dir && !isExpandedHtml);
 640                         }
 641                 });
 642
 643                 if (pgFiles != null) {
 644                         pgFiles.setMinMax(0, infoFilesAndSubdirs.length);
 645                 }
 646
 647                 for (File infoFileOrSubdir : infoFilesAndSubdirs) {
 648                         if (pgFiles != null) {
 649                                 pgFiles.setName(infoFileOrSubdir.getName());
 650                         }
 651
 652                         if (infoFileOrSubdir.isDirectory()) {
 653                                 addToStories(null, infoFileOrSubdir);
 654                         } else {
 655                                 try {
 656                                         MetaData meta = InfoReader
 657                                                         .readMeta(infoFileOrSubdir, false);
 658                                         try {
 659                                                 int id = Integer.parseInt(meta.getLuid());
 660                                                 if (id > lastId) {
 661                                                         lastId = id;
 662                                                 }
 663
 664                                                 stories.put(meta, new File[] { infoFileOrSubdir,
 665                                                                 getTargetFile(meta, infoFileOrSubdir) });
 666                                         } catch (Exception e) {
 667                                                 // not normal!!
 668                                                 throw new IOException("Cannot understand the LUID of "
 669                                                                 + infoFileOrSubdir + ": " + meta.getLuid(), e);
 670                                         }
 671                                 } catch (IOException e) {
 672                                         // We should not have not-supported files in the
 673                                         // library
 674                                         Instance.getInstance().getTraceHandler()
 675                                                         .error(new IOException("Cannot load file from library: " + infoFileOrSubdir, e));
 676                                 }
 677                         }
 678
 679                         if (pgFiles != null) {
 680                                 pgFiles.add(1);
 681                         }
 682                 }
 683         }
 684 }