private static String getHTML(String strURL)
{
String scrappingResult = String.Empty;
try
{
if (strURL != String.Empty)
{
if (setAllowUnsafeHeaderParsing())
{
System.Net.HttpWebRequest wReq = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(strURL);
wReq.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)";
using (System.Net.HttpWebResponse wResp = (System.Net.HttpWebResponse)wReq.GetResponse())
{
using (System.IO.StreamReader oSRead = new System.IO.StreamReader(wResp.GetResponseStream()))
scrappingResult = oSRead.ReadToEnd();
}
wReq.Abort();
}
}
}
catch (Exception e)
{
//gbl.ErrorLog(e.Message, "GetHTML", Global.scrapping_data);
scrappingResult = String.Empty;
}
return scrappingResult;
}
private static bool setAllowUnsafeHeaderParsing()
{
System.Reflection.Assembly aNetAssembly = System.Reflection.Assembly.GetAssembly(typeof(System.Net.Configuration.SettingsSection));
if (aNetAssembly != null)
{
Type aSettingsType = aNetAssembly.GetType("System.Net.Configuration.SettingsSectionInternal");
if (aSettingsType != null)
{
object anInstance = aSettingsType.InvokeMember("Section",
System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.GetProperty | System.Reflection.BindingFlags.NonPublic, null, null, new object[] { });
if (anInstance != null)
{
System.Reflection.FieldInfo aUseUnsafeHeaderParsing = aSettingsType.GetField("useUnsafeHeaderParsing", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance);
if (aUseUnsafeHeaderParsing != null)
{
aUseUnsafeHeaderParsing.SetValue(anInstance, true);
return true;
}
}
}
}
return false;
}
