爬虫
************** 反射和泛型写的增删改查******************************
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using ConsoleApplication1.Model;
using System.Data;
using System.Data.SqlClient;
using System.Reflection;
using System.Configuration;
namespace ConsoleApplication1.DAL
{
public class DbHelper
{
/// <summary>
/// 根据ID查看
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="id"></param>
/// <returns></returns>
public T GetModel<T>(int id)
{
Type type = typeof(T); //获取传进来对像
string properties = "";
//获取对象的属性,
foreach (PropertyInfo item in type.GetProperties())
{
properties += "[" + item.Name + "],";
}
properties = properties.Remove(properties.Length - 1, 1); //删除多余的逗号
string propertyFirst = type.GetProperties()[0].Name;//获取第一个数据
//链接数据库，得到数据
string sqlStr = "select top 1 " + properties + " from [" + type.Name + "] where [" + propertyFirst + "]=" + id + "";
SqlDataAdapter db = new SqlDataAdapter(sqlStr, new SqlConnection(ConfigurationManager.AppSettings["conn"]));
DataSet ds = new DataSet();
db.Fill(ds);
T t = (T)Activator.CreateInstance(type);//实例化对象
foreach (PropertyInfo item in type.GetProperties())
{
//得到属性的名字
item.SetValue(t, ds.Tables[0].Rows[0][item.Name]);
}
return t;
}
//有条件的查全表
public DataSet GetList<T>(string strWhere)
{
Type type = typeof(T);
StringBuilder str = new StringBuilder();
string properties = "";
foreach (PropertyInfo item in type.GetProperties())
{
properties += "[" + item.Name + "],";
}
properties = properties.Remove(properties.Length - 1, 1);//删除最后多余的逗号
str.Append("select " + properties + " ");
str.Append(" FROM [" + type.Name + "] ");
if (strWhere.Trim() != "")
{
str.Append(" where " + strWhere);
}
SqlDataAdapter sda = new SqlDataAdapter(str.ToString(), new SqlConnection(ConfigurationManager.AppSettings["conn"]));
DataSet ds = new DataSet();
sda.Fill(ds);
return ds;
}
/// <summary>
/// 添加数据
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="t"></param>
/// <returns></returns>
public int Add<T>(T t)
{
StringBuilder str = new StringBuilder();
str.Append("insert into");
//table() values( ?);select @@IDENTITY");
Type type = typeof(T);
str.Append(" [" + type.Name + "] ");
str.Append("( ");
for (int i = 1; i < type.GetProperties().Length; i++)
{
PropertyInfo item = type.GetProperties()[i];
str.Append(" [" + item.Name + "],");
}
str.Remove(str.Length - 1, 1);//去调逗号
str.Append(") values(");
//----------------------
for (int i = 1; i < type.GetProperties().Length; i++)
{
PropertyInfo item = type.GetProperties()[i];
str.Append(" '" + item.GetValue(t) + "',");
}
str.Remove(str.Length - 1, 1);
str.Append(");select @@IDENTITY");
SqlConnection conn = new SqlConnection(ConfigurationManager.AppSettings["conn"]);
SqlCommand cmd = new SqlCommand(str.ToString(), conn);
conn.Open();
object obj = cmd.ExecuteScalar();
if (obj != null)
{
return Convert.ToInt32(obj);
}
else
{
return 0;
}
}
/// <summary>
/// 增加一条数据
/// </summary>
///
public int ADD<T>(T t)
{
Type type = typeof(T); //对象
Type tp = t.GetType();
string val = "";
foreach (PropertyInfo item in tp.GetProperties())
{
val += "'" + tp.GetProperty(item.Name).GetValue(t, null) + "',";
}
val = val.Remove(val.Length - 1, 1);
val = val.Remove(1, 4);
string sqlStr = "insert into " + type.Name + " values(" + val + ");select @@IDENTITY";
SqlConnection conn = new SqlConnection(ConfigurationManager.AppSettings["conn"]);
SqlCommand cmd = new SqlCommand(sqlStr, conn);
conn.Open();
int flag = Convert.ToInt32(cmd.ExecuteScalar());
conn.Close();
return flag;
}
/// <summary>
/// 修改数据
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="t"></param>
/// <returns></returns>
public bool Update<T>(T t, int id)
{
StringBuilder strSql = new StringBuilder();
Type type = typeof(T);
strSql.Append("update [" + type.Name + "] set ");
for (int i = 1; i < type.GetProperties().Length; i++)
{
strSql.Append("" + type.GetProperties()[i].Name + "='" + type.GetProperties()[i].GetValue(t) + "',");
}
strSql = strSql.Remove(strSql.Length - 1, 1);
string PropertiesFirst = type.GetProperties()[0].Name;
int PropertiesFirstValue = (int)type.GetProperties()[0].GetValue(t);
strSql.Append(" where " + PropertiesFirst + "=" + id + "");
SqlConnection conn = new SqlConnection(ConfigurationManager.AppSettings["conn"]);
conn.Open();
SqlCommand cmd = new SqlCommand(strSql.ToString(), conn);
int rows = cmd.ExecuteNonQuery();
conn.Close();
if (rows > 0)
{
return true;
}
else
{
return false;
}
}
/// <summary>
///修改数据
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="t"></param>
/// <returns></returns>
public bool UpdateData<T>(T t)
{
Type type = typeof(T);
Type t1 = t.GetType();
//获取表中的字段（属性）
string shuxing = "";
foreach (PropertyInfo item in type.GetProperties())
{ shuxing += "" + item.Name + ","; }
shuxing = shuxing.Remove(shuxing.Length - 1, 1);
string [] s = shuxing.Split(',');
//获取t中的值
string val = "";
foreach (PropertyInfo i in t1.GetProperties())
{
val += "'" + t1.GetProperty(i.Name).GetValue(t, null) + "',";
}
val = val.Remove(val.Length - 1, 1);
string [] v = val.Split(',');
//拼接字段
string s1 = "";
for (int i = 1; i < s.Length; i++) { s1 += "" + s[i] + " =" + v[i] + ","; }
s1 = s1.Remove(s1.Length - 1, 1);
StringBuilder str = new StringBuilder();
str.Append("update ["+type.Name+"] set ");
str.Append(""+s1+"");
str.Append(" where "+s[0]+" = "+v[0]+"");
SqlConnection conn = new SqlConnection(ConfigurationManager.AppSettings["conn"]);
SqlCommand cmd = new SqlCommand(str.ToString(), conn);
conn.Open();
int flag = Convert.ToInt32(cmd.ExecuteNonQuery());
conn.Close();
if (flag > 0) { return true; } else { return false; }
}
/// <summary>
/// 删除
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="ID"></param>
/// <returns></returns
public int Delete<T>(int id)
{
Type type = typeof(T);
string ID = type.GetProperties()[0].Name;
string sql = "delete from " + type.Name + " where " + ID + " = " + id + "";
SqlConnection conn = new SqlConnection(ConfigurationManager.AppSettings["conn"]);
SqlCommand cmd = new SqlCommand(sql, conn);
conn.Open();
int flag = cmd.ExecuteNonQuery();
conn.Close();
return flag;
}
}
}
******************** 简单爬虫*****
//前台
<asp:Button ID="Button1" runat="server" Text="抓取" OnClick="Button1_Click" />
//后台
using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Web.UI;
using System.Web.UI.WebControls;
using System.Configuration;
using System.Data.SqlClient;
using System.Data;
using System.Reflection;
using ConsoleApplication1.Model;
using ConsoleApplication1.DAL;
using HtmlAgilityPack; //////////////引用第三类库，Install-Package HtmlAgilityPack -Version 1.11.2
public partial class ReptileText : System.Web.UI.Page
{
protected void Page_Load(object sender, EventArgs e)
{
}
protected void Button1_Click(object sender, EventArgs e)
{
DbHelper db = new DbHelper(); /// 通用增删改查的类名
HtmlWeb webClient = new HtmlWeb(); /////////model类，表名
HtmlAgilityPack.HtmlDocument doc = webClient.Load("https://ly.esf.fang.com/house-a010204-b012374/");
HtmlNodeCollection html = doc.DocumentNode.SelectNodes(".//dl[@class='clearfix']");
HtmlNodeCollection nn = doc.DocumentNode.SelectNodes(".//span[@class='tit_shop']");
HtmlNodeCollection price = doc.DocumentNode.SelectNodes(".//span[@class='red']");
Table_BC home = new Table_BC();
if (html != null)
{
for (int i = 1; i < html.Count;i++ )
{
HtmlNode name = doc.DocumentNode.SelectSingleNode(".//dl[@class='clearfix'][" + i + "]/dd[1]/h4[1]/a[1]/span[1]");
HtmlNode Prace = doc.DocumentNode.SelectSingleNode(".//dl[@class='clearfix'][" + i + "]/dd[2]/span[2]");
HtmlNode mm = doc.DocumentNode.SelectSingleNode(".//dl[@class='clearfix'][" + i + "]/dd[3]/span[3]");
if (name != null && Prace != null&&mm!=null)
{
///////////把抓到的数据。存到数据库，
home.Name = name.InnerText;
home.Title = Prace.InnerText;
home.MMNN = mm.InnerText;
db.ADD<Table_BC>(home);
}
}
}
}
}
*************************************----需要解压页面，爬虫************
{
string htmlCode;
HttpWebRequest webRequest = (
webRequest.Timeout = 30000;
webRequest.Method = "GET";
webRequest.UserAgent = "Mozilla/4.0";
webRequest.Headers.Add("Accept-Encoding", "gzip, deflate");
HttpWebResponse webResponse = (
if (webResponse.ContentEncoding.ToLower() == "gzip")//如果使用了GZip则先解压
{
using (
{
using (var zipStream =
new
{
Encoding enc = System.Text.Encoding.GetEncoding("gb2312");
using (StreamReader sr = new
{
htmlCode = sr.ReadToEnd();
}
}
}
}
else
{
using (
{
Encoding enc = System.Text.Encoding.GetEncoding("gb2312");
using (
{
htmlCode = sr.ReadToEnd();
}
}
}
return htmlCode;
}
//比特流 数据单位=》字符串 内存流Stream =>图片 MP3 MP4 BASE64
//byte[] btWeb = wc.DownloadData("
//string strWeb = System.Text.Encoding.GetEncoding("gb2312").GetString(btWeb);//gbk gb2312 utf-8 ASCII
string html = GetHtmlCode("
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(html);
HtmlNodeCollection dllist = doc.DocumentNode.SelectNodes("//dl[@class='clearfix']");
foreach (HtmlNode item in dllist)
{
}
int aa = dllist.Count;
//HtmlNodeCollection 节点列表 XMLNodeList
//HtmlAttributeCollection 属性列表
//HtmlNode节点 XMLNode
