Tuesday, February 7, 2012

Reading source code through website URL


private static String getHTML(String strURL)
    {
        String scrappingResult = String.Empty;
        try
        {
            if (strURL != String.Empty)
            {
                if (setAllowUnsafeHeaderParsing())
                {
                    System.Net.HttpWebRequest wReq = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(strURL);
                    wReq.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)";
                    using (System.Net.HttpWebResponse wResp = (System.Net.HttpWebResponse)wReq.GetResponse())
                    {
                        using (System.IO.StreamReader oSRead = new System.IO.StreamReader(wResp.GetResponseStream()))
                            scrappingResult = oSRead.ReadToEnd();
                    }
                    wReq.Abort();
                }
            }
        }
        catch (Exception e)
        {
            //gbl.ErrorLog(e.Message, "GetHTML", Global.scrapping_data);
            scrappingResult = String.Empty;
        }
        return scrappingResult;
    }

    private static bool setAllowUnsafeHeaderParsing()
    {
        System.Reflection.Assembly aNetAssembly = System.Reflection.Assembly.GetAssembly(typeof(System.Net.Configuration.SettingsSection));
        if (aNetAssembly != null)
        {
            Type aSettingsType = aNetAssembly.GetType("System.Net.Configuration.SettingsSectionInternal");
            if (aSettingsType != null)
            {
                object anInstance = aSettingsType.InvokeMember("Section",
                  System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.GetProperty | System.Reflection.BindingFlags.NonPublic, null, null, new object[] { });

                if (anInstance != null)
                {
                    System.Reflection.FieldInfo aUseUnsafeHeaderParsing = aSettingsType.GetField("useUnsafeHeaderParsing", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance);
                    if (aUseUnsafeHeaderParsing != null)
                    {
                        aUseUnsafeHeaderParsing.SetValue(anInstance, true);
                        return true;
                    }
                }
            }
        }
        return false;
    }

No comments:

Post a Comment