硬盘缓存方案DiskLruCache源代码解析
前面研究了LruCache,它作为如今用的最多的内存缓存方案已经在非常多开源缓存框架中使用。相同的还有硬盘缓存方案也就是DiskLruCache,通常的做法就是使用内存和硬盘二级缓存。
用法
1.存储:
DiskLruCache diskLruCache= open(File directory, int appVersion, int valueCount, long maxSize); DiskLruCache.Editor editor = diskLruCache.edit(key); OuputStream ouputStream = editor.newOutputStream(0);
然后往该ouputStream中写入收就可以。假设是写入字符串能够使用。事实上就是封装了往输出流中写的代码。
editor.set(int index, String value);
2.訪问
DiskLruCache.Snapshot snapShot = mDiskLruCache.get(key); if (snapShot != null) { InputStream is = snapShot.getInputStream(0); }
假设是文本,能够直接获取
snapShot.getString(int index)
3.close() 和open方法相应,用于关闭DiskLruCache里面的方法。journalWriter被置null。就不能再更新日志文件了。
4.delete() 删除缓存文件夹下的全部文件,用于清空缓存。
二级缓存的框架
综合前面的内存缓存LruCache和硬盘缓存DiskLruCache,内存和硬盘二级缓存的大概框架能够例如以下代码
Bitmap bitmap = getBitmap(generateKey(url)) if (bitmap == null) { downLoadFromNetOnAsyncTask(url); // set the bitmap from menory } Bitmap getBitmap(String key){ Bitmap bitmap = null; if ((bitmap = lruCache.get(key)) == null) { DiskLruCache.Snapshot snapShot = diskLruCache.get(key); if (snapShot != null) { InputStream is = snapShot.getInputStream(0); bitmap = BitmapFactory.decodeStream(is); lruCache.put(bitmap); } } return bitmap; } void downLoadFromNetOnAsyncTask(String url) { // download a picture via Thread DiskLruCache.Editor editor = diskLruCache.edit(generateKey(url)); OuputStream ouputStream = editor.newOutputStream(0); storeToMemoryAndDisk(); }
日志文件格式
This cache uses a journal file named "journal". A typical journal file
looks like this:
libcore.io.DiskLruCache
1
100
2
CLEAN 3400330d1dfc7f3f7f4b8d4d803dfcf6 832 21054
DIRTY 335c4c6028171cfddfbaae1a9c313c52
CLEAN 335c4c6028171cfddfbaae1a9c313c52 3934 2342
REMOVE 335c4c6028171cfddfbaae1a9c313c52
DIRTY 1ab96a171faeeee38496d8b330771a7a
CLEAN 1ab96a171faeeee38496d8b330771a7a 1600 234
READ 335c4c6028171cfddfbaae1a9c313c52
READ 3400330d1dfc7f3f7f4b8d4d803dfcf6
The first five lines of the journal form its header. They are the constant string "libcore.io.DiskLruCache", the disk cache's version,the application's version, the value count, and a blank line.
Each of the subsequent lines in the file is a record of the state of a cache entry. Each line contains space-separated values: a state, a key,and optional state-specific values.
- DIRTY lines track that an entry is actively being created or updated.Every successful DIRTY action should be followed by a CLEAN or REMOVE action. DIRTY lines without a matching CLEAN or REMOVE indicate thattemporary files may need to be deleted.
- CLEAN lines track a cache entry that has been successfully published and may be read. A publish line is followed by the lengths of each of its values.
- READ lines track accesses for LRU.
- REMOVE lines track entries that have been deleted.
源代码分析
public final class DiskLruCache implements Closeable { static final String JOURNAL_FILE = "journal"; static final String JOURNAL_FILE_TMP = "journal.tmp"; static final String MAGIC = "libcore.io.DiskLruCache"; static final String VERSION_1 = "1"; static final long ANY_SEQUENCE_NUMBER = -1; private static final String CLEAN = "CLEAN"; private static final String DIRTY = "DIRTY"; private static final String REMOVE = "REMOVE"; private static final String READ = "READ"; private final File directory; private final File journalFile; private final File journalFileTmp; private final int appVersion; private final long maxSize; // 一个键相应几个文件 private final int valueCount; private long size = 0; // 全局操作日志文件 private Writer journalWriter; private final LinkedHashMap<String, Entry> lruEntries = new LinkedHashMap<String, Entry>(0, 0.75f, true); private int redundantOpCount; // 用来标识被成功提交的序号 private long nextSequenceNumber = 0; // 用一个线程来处理在冗余操作大于2000后重构日志 private final ExecutorService executorService = new ThreadPoolExecutor(0, 1, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>()); private final Callable<Void> cleanupCallable = new Callable<Void>() { @Override public Void call() throws Exception { synchronized (DiskLruCache.this) { if (journalWriter == null) { return null; // closed } trimToSize(); if (journalRebuildRequired()) { rebuildJournal(); redundantOpCount = 0; } } return null; } }; private DiskLruCache(File directory, int appVersion, int valueCount, long maxSize) { this.directory = directory; this.appVersion = appVersion; this.journalFile = new File(directory, JOURNAL_FILE); this.journalFileTmp = new File(directory, JOURNAL_FILE_TMP); this.valueCount = valueCount; this.maxSize = maxSize; } / public static DiskLruCache open(File directory, int appVersion, int valueCount, long maxSize) throws IOException { if (maxSize <= 0) { throw new IllegalArgumentException("maxSize <= 0"); } if (valueCount <= 0) { throw new IllegalArgumentException("valueCount <= 0"); } // prefer to pick up where we left off DiskLruCache cache = new DiskLruCache(directory, appVersion, valueCount, maxSize); // 日志文件存在即解析 if (cache.journalFile.exists()) { try { cache.readJournal(); cache.processJournal(); cache.journalWriter = new BufferedWriter(new FileWriter(cache.journalFile, true)); return cache; } catch (IOException journalIsCorrupt) { Libcore.logW("DiskLruCache " + directory + " is corrupt: " + journalIsCorrupt.getMessage() + ", removing"); cache.delete(); } } // 不存在则创建新的日志文件,然后重建日志文件。 directory.mkdirs(); cache = new DiskLruCache(directory, appVersion, valueCount, maxSize); cache.rebuildJournal(); return cache; } // 读日志文件 private void readJournal() throws IOException { InputStream in = new BufferedInputStream(new FileInputStream(journalFile)); try { String magic = Streams.readAsciiLine(in); String version = Streams.readAsciiLine(in); String appVersionString = Streams.readAsciiLine(in); String valueCountString = Streams.readAsciiLine(in); String blank = Streams.readAsciiLine(in); if (!MAGIC.equals(magic) || !VERSION_1.equals(version) || !Integer.toString(appVersion).equals(appVersionString) || !Integer.toString(valueCount).equals(valueCountString) || !"".equals(blank)) { throw new IOException("unexpected journal header: [" + magic + ", " + version + ", " + valueCountString + ", " + blank + "]"); } while (true) { try { // 循环读取每一行 readJournalLine(Streams.readAsciiLine(in)); } catch (EOFException endOfJournal) { break; } } } finally { IoUtils.closeQuietly(in); } } // 读每一行,依据每行的字符串构建Entry private void readJournalLine(String line) throws IOException { String[] parts = line.split(" "); if (parts.length < 2) { throw new IOException("unexpected journal line: " + line); } String key = parts[1]; if (parts[0].equals(REMOVE) && parts.length == 2) { lruEntries.remove(key); return; } // 假设存在对key的操作会使其移动到链表尾 Entry entry = lruEntries.get(key); if (entry == null) { // 假设不存在则加入 entry = new Entry(key); lruEntries.put(key, entry); } if (parts[0].equals(CLEAN) && parts.length == 2 + valueCount) { entry.readable = true; entry.currentEditor = null; entry.setLengths(Arrays.copyOfRange(parts, 2, parts.length)); } else if (parts[0].equals(DIRTY) && parts.length == 2) { entry.currentEditor = new Editor(entry); } else if (parts[0].equals(READ) && parts.length == 2) { // this work was already done by calling lruEntries.get() // 假设为READ则什么都不须要做。上面这句翻译一下就是说这里要做的工作已经在调用lruEntries.get()时做过了 // 遇到READ事实上就是再次訪问该key。因此上面调用get的时候已经将其移动到近期使用的位置了 } else { throw new IOException("unexpected journal line: " + line); } } // 处理日志。将加入到Map中的全部键所占的磁盘空间加起来。赋值给size private void processJournal() throws IOException { // 假设有日志文件的备份文件存在则删除它 deleteIfExists(journalFileTmp); for (Iterator<Entry> i = lruEntries.values().iterator(); i.hasNext();) { Entry entry = i.next(); if (entry.currentEditor == null) { for (int t = 0; t < valueCount; t++) { size += entry.lengths[t]; } } else { // 当前条目正在被编辑,删除正在编辑的文件并将currentEditor赋值为null entry.currentEditor = null; for (int t = 0; t < valueCount; t++) { deleteIfExists(entry.getCleanFile(t)); deleteIfExists(entry.getDirtyFile(t)); } i.remove(); } } } // 创建新的日志文件,忽略掉冗余的信息,也就是对hashmap里面的元素进行遍历又一次写日志文件 private synchronized void rebuildJournal() throws IOException { if (journalWriter != null) { journalWriter.close(); } // 创建暂时日志文件 Writer writer = new BufferedWriter(new FileWriter(journalFileTmp)); writer.write(MAGIC); writer.write("\n"); writer.write(VERSION_1); writer.write("\n"); writer.write(Integer.toString(appVersion)); writer.write("\n"); writer.write(Integer.toString(valueCount)); writer.write("\n"); writer.write("\n"); // 遍历Map写入日志文件 for (Entry entry : lruEntries.values()) { if (entry.currentEditor != null) { writer.write(DIRTY + ' ' + entry.key + '\n'); } else { writer.write(CLEAN + ' ' + entry.key + entry.getLengths() + '\n'); } } writer.close(); // 重命名 journalFileTmp.renameTo(journalFile); journalWriter = new BufferedWriter(new FileWriter(journalFile, true)); } private static void deleteIfExists(File file) throws IOException { Libcore.deleteIfExists(file); } // 获取某个key相应的快照。通过该快照能够恢复到该缓存的文件 public synchronized Snapshot get(String key) throws IOException { checkNotClosed(); validateKey(key); // 对它的訪问会让他移动到Map的近期使用过的位置 Entry entry = lruEntries.get(key); if (entry == null) { return null; } if (!entry.readable) { return null; } /* * Open all streams eagerly to guarantee that we see a single published * snapshot. If we opened streams lazily then the streams could come * from different edits. */ InputStream[] ins = new InputStream[valueCount]; try { for (int i = 0; i < valueCount; i++) { // 通过干净的文件创建的流。于是假设获取某个键相应的文件还未被编辑完毕,文件不存在, // 那么返回null,因此能够在从硬盘获取时依据返回值是否为null推断硬盘是否有该缓存 ins[i] = new FileInputStream(entry.getCleanFile(i)); } } catch (FileNotFoundException e) { // a file must have been deleted manually! return null; } redundantOpCount++; journalWriter.append(READ + ' ' + key + '\n'); if (journalRebuildRequired()) { executorService.submit(cleanupCallable); } return new Snapshot(key, entry.sequenceNumber, ins); } /** * Returns an editor for the entry named {@code key}, or null if another * edit is in progress. */ public Editor edit(String key) throws IOException { return edit(key, ANY_SEQUENCE_NUMBER); } // 当DiskLruCache类调用edit方法传入的都是ANY_SEQUENCE_NUMBER。Snapshot调用edit的时候入当前Entry的序列号 // 目的是当Snapshot调用edit的时候假设该条目的序列号已经改变了(在持有这个Snapshot后又成功commit了)就会返回null // 这也就是为什么Snapshot命名为快照的含义。 private synchronized Editor edit(String key, long expectedSequenceNumber) throws IOException { checkNotClosed(); validateKey(key); Entry entry = lruEntries.get(key); if (expectedSequenceNumber != ANY_SEQUENCE_NUMBER && (entry == null || entry.sequenceNumber != expectedSequenceNumber)) { return null; // snapshot is stale } if (entry == null) { entry = new Entry(key); // 仅仅要edit,就put进去,这样在commit的时候假设出错就lrucache.remove,而且加入REMOVE记录。// 成功不用再put。
除了put进HashMap中的记录,其余的全部属于冗余的。其它版本号用总行数-lrucache.size lruEntries.put(key, entry); } else if (entry.currentEditor != null) { return null; // another edit is in progress } Editor editor = new Editor(entry); entry.currentEditor = editor; // 写入一条脏数记录 journalWriter.write(DIRTY + ' ' + key + '\n'); journalWriter.flush(); return editor; } public File getDirectory() { return directory; } public long maxSize() { return maxSize; } public synchronized long size() { return size; } private synchronized void completeEdit(Editor editor, boolean success) throws IOException { Entry entry = editor.entry; if (entry.currentEditor != editor) { throw new IllegalStateException(); } // readabe为false也即该条目还未被成功写入成为CLEAN。也就是首次创建缓存文件 // 也就是CLEAN必须和DIRTY配对。假设脏的文件不存在就出现异常edit didn't create file if (success && !entry.readable) { for (int i = 0; i < valueCount; i++) { if (!entry.getDirtyFile(i).exists()) { editor.abort(); // 一定要在同一个线程里面创建newOutputStream和提交commit,不然可能会出问题 throw new IllegalStateException("edit didn't create file " + i); } } } // 假设sucess也就是写I/O未出现异常那么将脏的文件重命名成干净的文件然后更新entry的中的文件大小字段,而且更新已经占用的磁盘空间size。 // 假设false也就是写I/O出错,就将脏的文件删除 for (int i = 0; i < valueCount; i++) { File dirty = entry.getDirtyFile(i); if (success) { if (dirty.exists()) { File clean = entry.getCleanFile(i); dirty.renameTo(clean); long oldLength = entry.lengths[i]; long newLength = clean.length(); entry.lengths[i] = newLength; size = size - oldLength + newLength; } } else { // 将脏的文件删除 deleteIfExists(dirty); } } redundantOpCount++; // 提交完毕后不论成功与否,将该条目的currentEditor置为null,以便其它地方能够在同一个key上进行再次edit entry.currentEditor = null; // readable为true指这个文件已经被成功创建过了。
// 有两地方对readable赋值为true。
一个是在解析日志文件时遇到clean时,此时即在再次訪问该条目。要写入到日志文件 // 还有一处就是这里。假设写入成功了success。那么将置为readble置为true。此时还要写入到日志文件 // 这里条件指要么之前该条目已经被成功的写入过,或者这次成功写入或者都为true if (entry.readable | success) { entry.readable = true; journalWriter.write(CLEAN + ' ' + entry.key + entry.getLengths() + '\n'); if (success) { // 每次提交成功就又一次赋值该Entry的序列号加一。
entry.sequenceNumber = nextSequenceNumber++; } } else { // 首次写入而且写入失败 lruEntries.remove(entry.key); journalWriter.write(REMOVE + ' ' + entry.key + '\n'); } // 推断是否达到重构日志的条件,运行之。 if (size > maxSize || journalRebuildRequired()) { // 在一个新的线程里面重构日志 executorService.submit(cleanupCallable); } } // 推断是否达到重构日志的要求:冗余操作大于有效数据的数目而且大于2000 private boolean journalRebuildRequired() { final int redundantOpCompactThreshold = 2000; return redundantOpCount >= redundantOpCompactThreshold && redundantOpCount >= lruEntries.size(); } /** * Drops the entry for {@code key} if it exists and can be removed. Entries * actively being edited cannot be removed. * * @return true if an entry was removed. */ public synchronized boolean remove(String key) throws IOException { checkNotClosed(); validateKey(key); Entry entry = lruEntries.get(key); // 正处于编辑状态还未被提交则不能被remove if (entry == null || entry.currentEditor != null) { return false; } // 循环删除相应的文件 for (int i = 0; i < valueCount; i++) { File file = entry.getCleanFile(i); if (!file.delete()) { throw new IOException("failed to delete " + file); } size -= entry.lengths[i]; entry.lengths[i] = 0; } redundantOpCount++; // 加入REMOVE记录 journalWriter.append(REMOVE + ' ' + key + '\n'); // 从LinkedHashMap中移除 lruEntries.remove(key); // 推断重构条件 if (journalRebuildRequired()) { executorService.submit(cleanupCallable); } return true; } public boolean isClosed() { return journalWriter == null; } private void checkNotClosed() { if (journalWriter == null) { throw new IllegalStateException("cache is closed"); } } public synchronized void flush() throws IOException { checkNotClosed(); trimToSize(); journalWriter.flush(); } // 关闭DiskLruCache public synchronized void close() throws IOException { if (journalWriter == null) { return; // already closed } // 终止全部的正处于编辑状态的条目 for (Entry entry : new ArrayList<Entry>(lruEntries.values())) { if (entry.currentEditor != null) { entry.currentEditor.abort(); } } // 删除近期最少使用的条目 trimToSize(); journalWriter.close(); journalWriter = null; } // 删除近期最少使用的条目,包含从Map删除和从磁盘删除文件 private void trimToSize() throws IOException { while (size > maxSize) { Map.Entry<String, Entry> toEvict = lruEntries.entrySet().iterator().next(); remove(toEvict.getKey()); } } // 删除全部的缓存文件。清除缓存用 public void delete() throws IOException { close(); IoUtils.deleteContents(directory); } private void validateKey(String key) { if (key.contains(" ") || key.contains("\n") || key.contains("\r")) { throw new IllegalArgumentException( "keys must not contain spaces or newlines: \"" + key + "\""); } } private static String inputStreamToString(InputStream in) throws IOException { return Streams.readFully(new InputStreamReader(in, Charsets.UTF_8)); } // DiskLruCache的get方法返回的对象的封装 // 事实上和Entry差点儿一样的。
为什么要从新封装一个Snapshot类呢? // 1.这个类里面封装了相应的缓存的文件的流,通过该流能够读取到里面的数据 // 2.这里面封装的sequenceNumber用于推断该快照是否已经过期。获取到该快照后。假设该key又经过了commit, // 它的序列号已经添加,于是该快照就过期了。调用edit方法返回null public final class Snapshot implements Closeable { // 和Entry中的key一样 private final String key; // 顾名思义是序列号的意思。这个值是在调用get方法获得快照的时候从Entry里面读取的。 private final long sequenceNumber; // 相应的输入流的数组,通过它能够读取到文件中面的数据 private final InputStream[] ins; private Snapshot(String key, long sequenceNumber, InputStream[] ins) { this.key = key; this.sequenceNumber = sequenceNumber; this.ins = ins; } // 通过快照过去对该条目的editor,能够有以下情况 // 1.假设是已经成功编辑完的则获取一个新的editor // 2.该快照在创建后已经被改变了,返回null // 3.还有一个editor正在编辑还未commit。返回null public Editor edit() throws IOException { return DiskLruCache.this.edit(key, sequenceNumber); } /** * Returns the unbuffered stream with the value for {@code index}. */ // 获取xxx.index文件相应的流 public InputStream getInputStream(int index) { return ins[index]; } // 假设某个index的流里面是文本,则通过getString获得文本 public String getString(int index) throws IOException { return inputStreamToString(getInputStream(index)); } @Override public void close() { for (InputStream in : ins) { IoUtils.closeQuietly(in); } } } // 通过DiskLruCache的edit方法获取该Editor对象。用来完毕对Entry的编辑 public final class Editor { // 每一个editor编辑一个entry条目 private final Entry entry; // 标识在I/O过程中是否有发生错误 private boolean hasErrors; private Editor(Entry entry) { this.entry = entry; } // 获取某个xxx.index文件的流,即用来读取改文件 public InputStream newInputStream(int index) throws IOException { synchronized (DiskLruCache.this) { if (entry.currentEditor != this) { throw new IllegalStateException(); } if (!entry.readable) { return null; } return new FileInputStream(entry.getCleanFile(index)); } } // 以String的方式获取最上一次提交的值。也就是获取该条目中的xxx.index文件的内容。
// 一般用于改文件保存字符串时使用 public String getString(int index) throws IOException { InputStream in = newInputStream(index); return in != null ? inputStreamToString(in) : null; } // 返回一个OutputStream,也就是被封装成FaultHidingOutputStream。不会再抛出I/O异常 // The returned output stream does not throw IOExceptions. public OutputStream newOutputStream(int index) throws IOException { synchronized (DiskLruCache.this) { if (entry.currentEditor != this) { throw new IllegalStateException(); } // 注意这里用的是getDirtyFile return new FaultHidingOutputStream(new FileOutputStream(entry.getDirtyFile(index))); } } // 用于往xx.index.tmp文件中面写文本 // 一般用于改文件保存字符串时使用 // 事实上仅仅是在上面newOutputStream的基础上添加了往文件中面写数据的代码,毕竟写几个字符串简单嘛 // 假设是从网络获取较大的图片或者文件,还是自己拿到这个OutputStream然后网里面写比較好 public void set(int index, String value) throws IOException { Writer writer = null; try { writer = new OutputStreamWriter(newOutputStream(index), Charsets.UTF_8); writer.write(value); } finally { IoUtils.closeQuietly(writer); } } /** * Commits this edit so it is visible to readers. This releases the * edit lock so another edit may be started on the same key. */ // 每次edit后写完数据须要调用commit函数 // 调用commit的原因: // 1.依据流的写过程是否出错调用completeEdit函数运行不同的逻辑 // 2.在completeEdit函数里最重要的一点是entry.currentEditor = null;还有一个edit才干在同一个key上编辑写入。这一点參考323行。每次调用edit方法假设在该key相应的条目的currentEditor部位null说明有一个edit正在编辑它,就要返回null,即如今不同意在对他进行编辑 public void commit() throws IOException { if (hasErrors) { // 出现I/O错误 completeEdit(this, false); remove(entry.key); // the previous entry is stale } else { // 正常情况 completeEdit(this, true); } } // 终止edit数据 public void abort() throws IOException { completeEdit(this, false); } // 封装的OutputStream,优点是屏蔽掉全部可能出现I/O异常的地方 // 假设出现I/O异常则将hasErrors赋值为false,这样后面处理逻辑简单 // 仅仅须要将其理解为不抛出异常而是置位hasErroes标志的OutputStream就可以 private final class FaultHidingOutputStream extends FilterOutputStream { private FaultHidingOutputStream(OutputStream out) { super(out); } @Override public void write(int oneByte) { try { out.write(oneByte); } catch (IOException e) { hasErrors = true; } } @Override public void write(byte[] buffer, int offset, int length) { try { out.write(buffer, offset, length); } catch (IOException e) { hasErrors = true; } } @Override public void close() { try { out.close(); } catch (IOException e) { hasErrors = true; } } @Override public void flush() { try { out.flush(); } catch (IOException e) { hasErrors = true; } } } } // LinkedHashMap中的value条目,封装了一些简单的信息 private final class Entry { // key。通常是url的MD5 private final String key; // 所相应的文件的大小的数组,如:12876 1567 private final long[] lengths; // 假设该条目被提交过一次即为true private boolean readable; // 该条目所相应的editor。假设正在被edit不为null,否则(已经被写完了。commited)为null private Editor currentEditor; /** The sequence number of the most recently committed edit to this entry. */ private long sequenceNumber; private Entry(String key) { this.key = key; this.lengths = new long[valueCount]; } // 获取该条目相应的文件的大小的数组的字符串,用于写journal public String getLengths() throws IOException { StringBuilder result = new StringBuilder(); for (long size : lengths) { result.append(' ').append(size); } return result.toString(); } //读取journal文件,解析每行的条目填充该entry private void setLengths(String[] strings) throws IOException { if (strings.length != valueCount) { throw invalidLengths(strings); } try { for (int i = 0; i < strings.length; i++) { lengths[i] = Long.parseLong(strings[i]); } } catch (NumberFormatException e) { throw invalidLengths(strings); } } // 当journal文件的某一行所相应的文件的个数的和valueCount不匹配时调用 private IOException invalidLengths(String[] strings) throws IOException { throw new IOException("unexpected journal line: " + Arrays.toString(strings)); } // 获取“干净的”文件:xxx.i public File getCleanFile(int i) { return new File(directory, key + "." + i); } // 获取“脏的”文件:xxx.i.tmp public File getDirtyFile(int i) { return new File(directory, key + "." + i + ".tmp"); } } }