httpClient解析网站url得到内容保存至html（编码格式自动转换）

langxiashahai

上代码
public class BaseParserHtml {public static String ENCODE = "gb2312";public static String getHtmlContent(String url) {HttpClient httpClient = new HttpClient();HttpMethod get = new GetMethod();try {get.setURI(new URI(url, true));get.setRequestHeader("User-Agent","Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; zh-CN; rv:1.9.2) Gecko/20100115 Firefox/3.6");get.setRequestHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");httpClient.executeMethod(get);byte b[] = new byte[4096];java.io.InputStream is = get.getResponseBodyAsStream();int read = 0;java.io.ByteArrayOutputStream fos = new java.io.ByteArrayOutputStream();while ((read = is.read(b)) != -1) {fos.write(b, 0, read);}String tmp_char = get.getResponseHeader("Content-Type").getValue();if (tmp_char != null) {tmp_char = findCharSet(fos.toString());}if (tmp_char == null) {tmp_char = "utf-8";}String content = fos.toString(tmp_char);return content;} catch (Exception ex) {ex.printStackTrace();} finally {get.releaseConnection();}return null;}public static String findCharSet(String str) {String tmp_char = str.toLowerCase();try {if (tmp_char != null) {int index = tmp_char.indexOf("charset");if (index >= 0) {tmp_char = tmp_char.substring(index);int index2 = tmp_char.indexOf("=");if (index2 >= 0) {String tmp = tmp_char.substring(0, index2);if (tmp != null && tmp.trim().equals("charset")) {tmp_char = tmp_char.substring(index2 + 1).trim();index2 = tmp_char.indexOf(" ");if (index2 > 0) {try {String _char = tmp_char.substring(0, index2);ENCODE = _char;Charset.forName(_char);return _char;} catch (Exception e) {}}index2 = tmp_char.indexOf("\"");if (index2 > 0) {try {String _char = tmp_char.substring(0, index2);ENCODE = _char;Charset.forName(_char);return _char;} catch (Exception e) {}}if (tmp_char.length() < 50) {try {String _char = tmp_char;ENCODE = _char;Charset.forName(_char);return _char;} catch (Exception e) {}}return null;}}}}} catch (Exception e) {}return null;}public static FileMsg getFileMsg() {StringBuilder path = new StringBuilder();path.append("e:\\");File file1;String str = null;Date todaytime = new Date();SimpleDateFormat date = new SimpleDateFormat("yyyyMM");SimpleDateFormat ttime = new SimpleDateFormat("yyyyMMddhhMMSS");String name = ttime.format(todaytime);String folder = date.format(todaytime);File file = new File(path.toString());File[] files = file.listFiles();for (int i = 0; i < files.length; i++) {if (files[i].isDirectory()) {if (files[i].getName().equals(folder)) {str = files[i].getName();}}}if (str == null) {file1 = new File(path.append("\\" + folder + "\\").toString());file1.mkdir();} else {path.append("\\" + str + "\\");}FileMsg fm = new FileMsg();fm.setFolder(folder);fm.setName(name);fm.setPath(path);return fm;}public static String saveHtml(String fileString) {String newPath = "";FileMsg fm = getFileMsg();File file2 = new File(fm.getPath().append(fm.getName() + ".html").toString());FileOutputStream fos = null;OutputStreamWriter osw = null;PrintWriter outFile = null;try {file2.createNewFile();fos = new FileOutputStream(file2);osw = new OutputStreamWriter(fos, ENCODE);outFile = new PrintWriter(osw);outFile.write(fileString);newPath = fm.getPath().toString().replace("\\", "/");} catch (FileNotFoundException e) {e.printStackTrace();} catch (IOException e) {e.printStackTrace();} finally {try {osw.close();} catch (IOException e) {e.printStackTrace();}try {fos.close();} catch (IOException e) {e.printStackTrace();}}return newPath;}public void message(String szMsg) {try {System.out.println(new String(szMsg.getBytes(ENCODE), System.getProperty("file.encoding")));} catch (Exception ex) {ex.printStackTrace();}}}


public class FileMsg {private StringBuilder path;private String name;private String folder;public StringBuilder getPath() {return path;}public void setPath(StringBuilder path) {this.path = path;}public String getName() {return name;}public void setName(String name) {this.name = name;}public String getFolder() {return folder;}public void setFolder(String folder) {this.folder = folder;}}

public class Test {/** * @param args */public static void main(String[] args) {// TODO Auto-generated method stubSystem.setProperty( "org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog" );BaseParserHtml bph = new BaseParserHtml();String fileString = bph.getHtmlContent("http://www.mpshow.com.cn/product-9306.aspx");System.out.println("path:" + bph.saveHtml(fileString));}}