Java正确判别出文件的字符集-带BOM和不带BOM的UTF-8字符
Java正确判别出文件的字符集（尤其是带BOM和不带BOM的UTF-8字符)
前几天在项目中需要读取用户上传过来的txt文件，但不确定txt文件的字符集
UTF-16、UTF-8(带BOM)、Unicode可以根据前三个字节区别
public String getTxtEncode(FileInputStream in) throws IOException{
byte[] head = new byte[3];
in.read(head);
String code = "GBK";
if (head[0] == -1 && head[1] == -2 )
code = "UTF-16";
if (head[0] == -2 && head[1] == -1 )
code = "Unicode";
//带BOM
if(head[0]==-17 && head[1]==-69 && head[2] ==-65)
code = "UTF-8";
if("Unicode".equals(code)){
code = "UTF-16";
}
return code;
} 但不带BOM的UTF-8和GBK前三个字节不确定，用以上方法无法区别
通过在google上搜索发现不带BOM的识别是Java遗留的一个bug，呵呵，终于找到根源了，Java提供了此bug的解决方案
package com.justsy.sts.utf8;
import java.io.*;
/**
* This inputstream will recognize unicode BOM marks and will skip bytes if
* getEncoding() method is called before any of the read(...) methods.
*
* Usage pattern: String enc = "ISO-8859-1"; // or NULL to use systemdefault
* FileInputStream fis = new FileInputStream(file); UnicodeInputStream uin = new
* UnicodeInputStream(fis, enc); enc = uin.getEncoding(); // check and skip
* possible BOM bytes InputStreamReader in; if (enc == null) in = new
* InputStreamReader(uin); else in = new InputStreamReader(uin, enc);
*/
public class UnicodeInputStream extends InputStream {
PushbackInputStream internalIn;
boolean isInited = false;
String defaultEnc;
String encoding;
private static final int BOM_SIZE = 4;
public UnicodeInputStream(InputStream in, String defaultEnc) {
internalIn = new PushbackInputStream(in, BOM_SIZE);
this.defaultEnc = defaultEnc;
}
public String getDefaultEncoding() {
return defaultEnc;
}
public String getEncoding() {
if (!isInited) {
try {
init();
} catch (IOException ex) {
IllegalStateException ise = new IllegalStateException(
"Init method failed.");
ise.initCause(ise);
throw ise;
}
}
return encoding;
}
/**
* Read-ahead four bytes and check for BOM marks. Extra bytes are unread
* back to the stream, only BOM bytes are skipped.
*/
protected void init() throws IOException {
if (isInited)
return;
byte bom[] = new byte[BOM_SIZE];
int n, unread;
n = internalIn.read(bom, 0, bom.length);
if ((bom[0] == (byte) 0x00) && (bom[1] == (byte) 0x00)
&& (bom[2] == (byte) 0xFE) && (bom[3] == (byte) 0xFF)) {
encoding = "UTF-32BE";
unread = n - 4;
} else if ((bom[0] == (byte) 0xFF) && (bom[1] == (byte) 0xFE)
&& (bom[2] == (byte) 0x00) && (bom[3] == (byte) 0x00)) {
encoding = "UTF-32LE";
unread = n - 4;
} else if ((bom[0] == (byte) 0xEF) && (bom[1] == (byte) 0xBB)
&& (bom[2] == (byte) 0xBF)) {
encoding = "UTF-8";
unread = n - 3;
} else if ((bom[0] == (byte) 0xFE) && (bom[1] == (byte) 0xFF)) {
encoding = "UTF-16BE";
unread = n - 2;
} else if ((bom[0] == (byte) 0xFF) && (bom[1] == (byte) 0xFE)) {
encoding = "UTF-16LE";
unread = n - 2;
} else {
// Unicode BOM mark not found, unread all bytes
encoding = defaultEnc;
unread = n;
}
// System.out.println("read=" + n + ", unread=" + unread);
if (unread > 0)
internalIn.unread(bom, (n - unread), unread);
isInited = true;
}
public void close() throws IOException {
// init();
isInited = true;
internalIn.close();
}
public int read() throws IOException {
// init();
isInited = true;
return internalIn.read();
}
} 通过使用上述InputStream类的实现可以正确的读取出不带BOM和带BOM的字符集
package com.justsy.sts.utf8;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
public class UTF8Test {
public static void main(String[] args) throws IOException {
File f = new File("D:"+File.separator+"Order.txt");
FileInputStream in = new FileInputStream(f);
String dc = Charset.defaultCharset().name();
UnicodeInputStream uin = new UnicodeInputStream(in,dc);
BufferedReader br = new BufferedReader(new InputStreamReader(uin));
String line = br.readLine();
while(line != null)
{
System.out.println(line);
line = br.readLine();
}
}
}
结合Java提供的方案，我们就可以比较完整的判别出各种字符集了
public String getTxtEncode(FileInputStream in) throws IOException{
String dc = Charset.defaultCharset().name();
UnicodeInputStream uin = new UnicodeInputStream(in,dc);
if("UTF-8".equals(uin.getEncoding())){
uin.close();
return "UTF-8";
}
uin.close();
byte[] head = new byte[3];
in.read(head);
String code = "GBK";
if (head[0] == -1 && head[1] == -2 )
code = "UTF-16";
if (head[0] == -2 && head[1] == -1 )
code = "Unicode";
//带BOM
if(head[0]==-17 && head[1]==-69 && head[2] ==-65)
code = "UTF-8";
if("Unicode".equals(code)){
code = "UTF-16";
}
return code;
} 