Get All URLs on a Page
链接:http://www.csharphelp.com/archives4/archive693.html
1
GetUrls urls = new GetUrls(); urls.RetrieveUrls("http://www.microsoft.com");
2![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)
3![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)
4
The class is listed below. Have fun!
5
//required namespaces
6
using System;
7
using System.Collections.Generic;
8
using System.Text;
9
using System.Net;
10
using System.IO;
11
using System.Text.RegularExpressions;
12![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)
13![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)
14
namespace FindAllUrls
15
{
16
class GetUrls
17
{
18![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
19
//public method called from your application
20
public void RetrieveUrls( string webPage )
21
{
22
GetAllUrls(RetrieveContent(webPage));
23
}
24![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
25
//get the content of the web page passed in
26
private string RetrieveContent(string webPage)
27
{
28
HttpWebResponse response = null;//used to get response
29
StreamReader respStream = null;//used to read response into string
30
try
31
{
32
//create a request object using the url passed in
33
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(webPage);
34
request.Timeout = 10000;
35![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
36
//go get a response from the page
37
response = (HttpWebResponse)request.GetResponse();
38
39
//create a streamreader object from the response
40
respStream = new StreamReader(response.GetResponseStream());
41![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
42
//get the contents of the page as a string and return it
43
return respStream.ReadToEnd();
44
}
45
catch (Exception ex)//houston we have a problem!
46
{
47
throw ex;
48
}
49
finally
50
{
51
//close it down, we're going home!
52
response.Close();
53
respStream.Close();
54
}
55
}
56
57
//using a regular expression, find all of the href or urls
58
//in the content of the page
59
private void GetAllUrls( string content )
60
{
61
//regular expression
62
string pattern = @"(?:href\s*=)(?:[\s""']*)(?!#|mailto|location.|javascript|.*css|.*this\.)(?
63
.*?)(?:[\s>""'])";
64
65
//Set up regex object
66
Regex RegExpr = new Regex(pattern, RegexOptions.IgnoreCase);
67![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
68
//get the first match
69
Match match = RegExpr.Match(content);
70![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
71
//loop through matches
72
while (match.Success)
73
{
74![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
75
//output the match info
76
Console.WriteLine("href match: " + match.Groups[0].Value);
77
WriteToLog("C:\matchlog.txt", "href match: " + match.Groups[0].Value + "\r\n");
78![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
79
Console.WriteLine("Url match: " + match.Groups[1].Value);
80
WriteToLog("C:\matchlog.txt", "Url | Location | mailto match: " + match.Groups[1].Value + "\r\n");
81
82
//get next match
83
match = match.NextMatch();
84
}
85
}
86![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
87
//Write to a log file
88
private void WriteToLog(string file, string message)
89
{
90
using (StreamWriter w = File.AppendText(file))
91
{
92
w.WriteLine(DateTime.Now.ToString() + ": " + message); w.Close();
93
}
94
}
95
}
96
}
97![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)
98![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)
![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)
2
![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)
3
![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)
4
![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)
5
![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)
6
![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)
7
![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)
8
![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)
9
![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)
10
![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)
11
![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)
12
![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)
13
![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)
14
![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)
15
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedBlockStart.gif)
16
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
17
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedSubBlockStart.gif)
18
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
19
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
20
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
21
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedSubBlockStart.gif)
22
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
23
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedSubBlockEnd.gif)
24
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
25
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
26
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
27
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedSubBlockStart.gif)
28
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
29
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
30
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
31
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedSubBlockStart.gif)
32
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
33
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
34
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
35
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
36
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
37
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
38
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
39
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
40
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
41
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
42
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
43
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
44
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedSubBlockEnd.gif)
45
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
46
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedSubBlockStart.gif)
47
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
48
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedSubBlockEnd.gif)
49
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
50
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedSubBlockStart.gif)
51
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
52
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
53
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
54
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedSubBlockEnd.gif)
55
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedSubBlockEnd.gif)
56
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
57
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
58
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
59
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
60
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedSubBlockStart.gif)
61
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
62
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
63
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
64
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
65
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
66
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
67
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
68
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
69
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
70
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
71
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
72
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
73
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedSubBlockStart.gif)
74
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
75
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
76
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
77
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
78
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
79
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
80
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
81
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
82
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
83
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
84
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedSubBlockEnd.gif)
85
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedSubBlockEnd.gif)
86
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
87
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
88
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
89
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedSubBlockStart.gif)
90
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
91
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedSubBlockStart.gif)
92
![](https://www.cnblogs.com/Images/OutliningIndicators/InBlock.gif)
93
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedSubBlockEnd.gif)
94
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedSubBlockEnd.gif)
95
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedSubBlockEnd.gif)
96
![](https://www.cnblogs.com/Images/OutliningIndicators/ExpandedBlockEnd.gif)
97
![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)
98
![](https://www.cnblogs.com/Images/OutliningIndicators/None.gif)