View Javadoc
1   package com.ljs.ootp.extract.html.loader;
2   
3   import com.google.common.base.Charsets;
4   import com.google.common.base.Throwables;
5   import com.google.common.cache.AbstractCache;
6   import com.google.common.io.Files;
7   import java.io.File;
8   import java.io.IOException;
9   import java.net.URLEncoder;
10  import java.nio.charset.Charset;
11  import java.util.concurrent.Callable;
12  import org.apache.commons.lang3.StringUtils;
13  import org.jsoup.Jsoup;
14  import org.jsoup.nodes.Document;
15  
16  /**
17   *
18   * @author lstephen
19   */
20  public final class DiskCachingLoader implements PageLoader {
21  
22      public static final String DEFAULT_CACHE_DIR = "c:/ootp/cache";
23  
24      private final DiskCache cache;
25  
26      private final PageLoader wrapped;
27  
28      private DiskCachingLoader(String dir, PageLoader wrapped) {
29          cache = DiskCache.cachingTo(dir);
30          this.wrapped = wrapped;
31      }
32  
33      @Override
34      public Document load(String url) {
35          return cache.get(url, PageLoaders.asCallable(wrapped, url));
36      }
37  
38      private static final class DiskCache
39          extends AbstractCache<String, Document> {
40  
41          private static final Charset CHARSET = Charsets.UTF_8;
42  
43          private final String dir;
44  
45          private DiskCache(String dir) {
46              super();
47              this.dir = dir;
48          }
49  
50          private File getFileForKey(Object key) {
51  
52              String pathName =
53                  StringUtils.substringBeforeLast(key.toString(), "/");
54  
55              try {
56                  File path = new File(
57                      dir,
58                      URLEncoder.encode(pathName, CHARSET.name()));
59  
60                  String fileName =
61                      StringUtils.substringAfterLast(key.toString(), "/");
62  
63                  File forKey = new File(
64                      path,
65                      URLEncoder.encode(fileName, CHARSET.name()));
66  
67                  Files.createParentDirs(forKey);
68  
69                  return forKey;
70              } catch (IOException e) {
71                  throw Throwables.propagate(e);
72              }
73          }
74  
75          @Override
76          public void put(String key, Document value) {
77              try {
78                  Files.write(value.outerHtml(), getFileForKey(key), CHARSET);
79              } catch (IOException e) {
80                  throw Throwables.propagate(e);
81              }
82          }
83  
84          @Override
85          public Document get(String key, Callable<? extends Document> callable) {
86              Document doc = getIfPresent(key);
87  
88              if (doc == null) {
89                  try {
90                      put(key, callable.call());
91                  } catch (Exception e) {
92                      throw Throwables.propagate(e);
93                  }
94                  return getIfPresent(key);
95              } else {
96                  return doc;
97              }
98          }
99  
100         @Override
101         public Document getIfPresent(Object key) {
102             File f = getFileForKey(key);
103 
104             if (f.exists()) {
105                 try {
106                     return Jsoup.parse(f, CHARSET.name());
107                 } catch (IOException e) {
108                     throw Throwables.propagate(e);
109                 }
110             } else {
111                 return null;
112             }
113         }
114 
115         public static DiskCache cachingTo(String dir) {
116             return new DiskCache(dir);
117         }
118     }
119 
120     public static DiskCachingLoader wrap(PageLoader loader) {
121         return create(DEFAULT_CACHE_DIR, loader);
122     }
123 
124     public static DiskCachingLoader create(String dir, PageLoader loader) {
125         return new DiskCachingLoader(dir, loader);
126     }
127 
128 }