页面修改

EduSpider.Utility/HttpHelper.cs

@@ -29,7 +29,8 @@ namespace EduSpider.Utility
                 SslProtocols = System.Security.Authentication.SslProtocols.Tls
             };
             var http = new HttpClient(handler);
-            GenerateHttpHeader(ref http);
+            //GenerateHttpHeader(ref http);
+            FirefoxHttpHeader(ref http);
             return http;
         }
 
@@ -63,6 +64,24 @@ namespace EduSpider.Utility
             http.DefaultRequestHeaders.Add("pragma", "no-cache");
         }
 
+        /// <summary>
+        /// 火狐Http请求
+        /// </summary>
+        /// <param name="http"></param>
+        private static void FirefoxHttpHeader(ref HttpClient http)
+        {
+            http.DefaultRequestHeaders.Add("Accept", "application/json, text/plain, */*");
+            http.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br");
+            http.DefaultRequestHeaders.Add("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2");
+            http.DefaultRequestHeaders.Add("Origin", "https://console.eeo.cn");
+            http.DefaultRequestHeaders.Add("Referer", "https://console.eeo.cn/saas/school/index.html");
+            http.DefaultRequestHeaders.Add("Sec-Fetch-Dest", "empty");
+            http.DefaultRequestHeaders.Add("Sec-Fetch-Mode", "cors");
+            http.DefaultRequestHeaders.Add("Sec-Fetch-Site", "same-origin");
+            http.DefaultRequestHeaders.Add("TE", "trailers");
+            http.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:104.0) Gecko/20100101 Firefox/104.0");
+        }
+
         /// <summary>
         /// 拼接Cookie
         /// </summary>

EduSpider.Utility/Plugin/LogHelper.cs

+using System;
+using System.IO;
+using System.Threading.Tasks;
+
+namespace EduSpider.Utility.Plugin
+{
+    /// <summary>
+    /// 日志帮助类
+    /// </summary>
+    public class LogHelper
+    {
+        private static readonly string logDir = Config.LogPath;
+        private static readonly string infoLogDir = Config.InofLogPath;
+        private static readonly string requestLogDir = Config.RequestLogPath;
+        private static readonly object objError = new object();
+        private static readonly object objInfo = new object();
+        private static readonly object objRequest = new object();
+
+        /// <summary>
+        /// 构造函数
+        /// </summary>
+        static LogHelper()
+        {
+            if (!Directory.Exists(logDir))
+                Directory.CreateDirectory(logDir);
+            if (!Directory.Exists(infoLogDir))
+                Directory.CreateDirectory(infoLogDir);
+            if (!Directory.Exists(requestLogDir))
+                Directory.CreateDirectory(requestLogDir);
+        }
+
+
+        /// <summary>
+        /// 写日志(异常日志)
+        /// </summary>
+        /// <param name="ex">异常内容</param>
+        public static void Write(Exception ex)
+        {
+            Write(ex, "");
+        }
+
+        /// <summary>
+        /// 写日志(异常日志)
+        /// </summary>
+        /// <param name="msg">信息</param>
+        public static void Write(string msg)
+        {
+            Write(null, msg);
+        }
+
+        /// <summary>
+        /// 写日志(异常日志)
+        /// </summary>
+        /// <param name="exception">异常信息</param>
+        /// <param name="otherMsg">其他信息</param>
+        public static void Write(Exception exception, string otherMsg)
+        {
+            Task.Run(() => WriteLog(exception, otherMsg, LogEnum.Error));
+        }
+
+        /// <summary>
+        /// 打印信息(记录信息)
+        /// </summary>
+        /// <param name="msg">信息</param>
+        public static void WriteInfo(string msg)
+        {
+            Task.Run(() => WriteLog(null, msg, LogEnum.Info));
+        }
+
+
+        /// <summary>
+        /// 写日志
+        /// </summary>
+        /// <param name="exception">异常信息</param>
+        /// <param name="otherMsg">其他信息</param>
+        /// <param name="logType">日志类型</param>
+        private static void WriteLog(Exception exception, string otherMsg, LogEnum logType)
+        {
+            string str = "";
+            try
+            {
+                str += string.Format(@"
+DateTime:{0}", DateTime.Now.ToString());
+                if (exception != null)
+                {
+                    if (exception.InnerException != null)
+                    {
+                        exception = exception.InnerException;
+                    }
+                    str += string.Format(@"
+Message:{0}
+StackTrace:
+{1}
+Source:{2}
+"
+ , exception.Message
+ , exception.StackTrace
+ , exception.Source
+
+                    );
+                }
+                str += string.Format(@"
+ExtMessage:{0}", otherMsg);
+                string filePath = "";
+                object lockObj = new object();
+                switch (logType)
+                {
+                    case LogEnum.Error:
+                        filePath = Path.Combine(logDir, DateTime.Now.ToString("yyyyMMdd") + ".txt");
+                        lockObj = objError;
+                        break;
+                    case LogEnum.Info:
+                        filePath = Path.Combine(infoLogDir, DateTime.Now.ToString("yyyyMMdd") + ".txt");
+                        lockObj = objInfo;
+                        break;
+                    case LogEnum.Request:
+                        filePath = Path.Combine(requestLogDir, DateTime.Now.ToString("yyyyMMdd") + ".txt");
+                        lockObj = objRequest;
+                        break;
+                }
+                lock (lockObj)
+                {
+                    StreamWriter sw = new StreamWriter(filePath, true);
+                    sw.WriteLine(str);
+                    sw.Close();
+                }
+            }
+            catch
+            {
+
+            }
+        }
+    }
+    /// <summary>
+    /// 日志枚举
+    /// </summary>
+    public enum LogEnum
+    {
+        /// <summary>
+        /// 错误日志
+        /// </summary>
+        Error = 1,
+        /// <summary>
+        /// 信息记录
+        /// </summary>
+        Info = 2,
+        /// <summary>
+        /// 接口请求
+        /// </summary>
+        Request = 3
+    }
+}

EduSpider/EduSpider.csproj

@@ -36,7 +36,7 @@
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="6.0.0" />
     <PackageReference Include="OpenCvSharp4.Windows" Version="4.5.5.20211231" />
     <PackageReference Include="Selenium.WebDriver" Version="4.1.0" />
-    <PackageReference Include="Selenium.WebDriver.ChromeDriver" Version="102.0.5005.6102" />
+    <PackageReference Include="Selenium.WebDriver.GeckoDriver" Version="0.31.0.1" />
     <PackageReference Include="Serilog.AspNetCore" Version="3.4.0" />
     <PackageReference Include="Serilog.Sinks.Console" Version="3.1.1" />
     <PackageReference Include="Serilog.Sinks.RollingFile" Version="3.3.0" />

EduSpider/Spiders/ClassInRule/ClassInAccountManager.cs

@@ -2,8 +2,11 @@
 using OpenCvSharp;
 using OpenQA.Selenium;
 using OpenQA.Selenium.Chrome;
+using OpenQA.Selenium.Firefox;
 using OpenQA.Selenium.Interactions;
+using OpenQA.Selenium.Support.UI;
 using System;
+using System.Drawing;
 using System.IO;
 using System.Net;
 using System.Net.Http;
@@ -67,32 +70,58 @@ namespace TicketSpider.Spiders.ClassInRule
             var flag = false;
             IWebElement inputEle;
             IWebElement pwdEle;
-            var option = new ChromeOptions();
+            var option = new FirefoxOptions();
             option.AddArgument("--incognito");
             string IsOpenChrome= VTX.FW.Helper.ConfigHelper.GetAppsettings("IsOpenChrome");
             if (IsOpenChrome == "1")
             {
-
+                option.AddArgument("window-size=1920,1080");
             }
             else
             {
-                option.AddArgument("headless");
+                //google隐身模式
+                //option.AddArgument("headless");
+                //火狐隐身模式
+                //option.AddArgument("--headless");
+
+                //option.AddArgument("--disable-gpu");
+                // option.AddArgument("ingore-certificate-errors");
             }
-            option.AddArgument("disable-infobars");
-            option.AddArgument("window-size=1920,1080");
-            option.AddArgument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36");
-            ChromeDriverService service = ChromeDriverService.CreateDefaultService(Environment.CurrentDirectory);
-            service.HideCommandPromptWindow = true;
+            //option.AddArgument("--headless");
+            option.AddArgument("--disable-infobars");
+            option.AddArgument("window-size=1920*1080");
+           // option.AddArgument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:104.0) Gecko/20100101 Firefox/104.0");
             option.PageLoadStrategy = PageLoadStrategy.Eager;
-            IWebDriver driver = new ChromeDriver(service, option);
+            FirefoxDriverService service = FirefoxDriverService.CreateDefaultService(Environment.CurrentDirectory);
+            service.HideCommandPromptWindow = true;
+          
+            IWebDriver driver = new FirefoxDriver(service,option);
 
             VTX.FW.Helper.LogHelper.WriteInfo("RunAsync", "开始登录");
             try
             {
                 driver.Navigate().GoToUrl($"https://www.eeo.cn/cn/login");
-                driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(15);
-                inputEle = driver.FindElement(By.Name("phoneNum"));
-                pwdEle = driver.FindElement(By.Name("loginPwd"));
+
+                ITimeouts timeouts = driver.Manage().Timeouts();
+                //设置查找元素最大超时时间为30秒
+                timeouts.ImplicitWait = new TimeSpan(0, 0, 30);
+                //设置页面操作最大超时时间为30秒
+                timeouts.PageLoad = new TimeSpan(0, 0, 30);
+                //设置脚本异步最大超时时间为30秒
+                timeouts.AsynchronousJavaScript = new TimeSpan(0, 0, 30);
+                //等待页面元素加载完成
+                //默认等待100秒
+                WebDriverWait wait = new WebDriverWait(driver, TimeSpan.FromSeconds(100));
+
+                //等待页面上按钮加载
+                inputEle = wait.Until((d) =>
+                {
+                    return d.FindElement(By.Name("phoneNum"));
+                });
+                pwdEle = wait.Until((d) =>
+                {
+                    return d.FindElement(By.Name("loginPwd"));
+                });
                 if (inputEle != null && pwdEle != null)
                 {
                     inputEle.SendKeys("18140082327");
@@ -108,7 +137,7 @@ namespace TicketSpider.Spiders.ClassInRule
                         loginCount++;
                         InfoHelper.WriteLine($"正在进行第{loginCount}次模拟登录...");
                         VTX.FW.Helper.LogHelper.WriteInfo("RunAsync", $"正在进行第{loginCount}次模拟登录...");
-                        var login = BeginSliderLoginHandler(ref driver);
+                        var login = FireFoxBeginSliderLoginHandler(ref driver);
                         if (login.IsCompleted)
                         {
                             driver.SwitchTo().DefaultContent();
@@ -116,14 +145,13 @@ namespace TicketSpider.Spiders.ClassInRule
                             var cookies = driver.Manage().Cookies.AllCookies;
                             InfoHelper.WriteLine($"操作完成，正在等待结果");
                             VTX.FW.Helper.LogHelper.WriteInfo("RunAsync", $"操作完成，正在等待结果");
-                            
                             //处理
                             foreach (var item in cookies)
                             {
                                 _loginCookies += item.Name + "=" + item.Value + ";";
                             }
                             _loginCookies = _loginCookies.Substring(0, _loginCookies.Length - 1);
-                            if (!string.IsNullOrEmpty(_loginCookies)&&_loginCookies.Contains("sensorsdata2015jssdkcross"))
+                            if (!string.IsNullOrEmpty(_loginCookies)&&_loginCookies.Contains("sensorsdata2015jssdkcross")&&_loginCookies.Contains("_eeos_traffic"))
                             {
                                 flag = true;
                                 break;
@@ -156,12 +184,99 @@ namespace TicketSpider.Spiders.ClassInRule
 
         }
 
+        public static Task FireFoxBeginSliderLoginHandler(ref IWebDriver driver)
+        {
+            //等待滑块加载
+            Thread.Sleep(5000);
+            driver.SwitchTo().Frame(driver.FindElement(By.Id("tcaptcha_iframe")));
+
+            ((IJavaScriptExecutor)driver).ExecuteScript("document.getElementById(\"slideBlock\").style.display = 'none';");
+            var bgPng = ((ITakesScreenshot)driver.FindElement(By.Id("slideBg"))).GetScreenshot();
+            ((IJavaScriptExecutor)driver).ExecuteScript("document.getElementById(\"slideBlock\").style.display = 'block';document.getElementById(\"slideBg\").style.opacity = 0;");
+            var blockPng = ((ITakesScreenshot)driver.FindElement(By.Id("slideBlock"))).GetScreenshot();
+            ((IJavaScriptExecutor)driver).ExecuteScript("document.getElementById(\"slideBg\").style.opacity = 1;");
+            var dirpath = $"{Environment.CurrentDirectory}\\geetest";
+            if (!Directory.Exists(dirpath))
+            {
+                Directory.CreateDirectory(dirpath);
+            }
+            var fileBgPath = $"{dirpath}\\{DateTime.Now.Ticks}.png";
+            var fileBlockPath = $"{dirpath}\\{DateTime.Now.Ticks }__1.png";
+            bgPng.SaveAsFile(fileBgPath, ScreenshotImageFormat.Png);
+            blockPng.SaveAsFile(fileBlockPath, ScreenshotImageFormat.Png);
+            var bgX = FireFoxGetBoundary(fileBgPath, fileBlockPath);
+   
+            InfoHelper.WriteLine($"读取背景缺口边界X坐标： {bgX}");
+            VTX.FW.Helper.LogHelper.WriteInfo("BeginSliderLoginHandler", $"读取背景缺口边界X坐标： {bgX}");
+
+            Actions action = new(driver);
+            var sliderButton = driver.FindElement(By.Id("tcaptcha_drag_button"));
+            if (sliderButton != null)
+            {
+                ((IJavaScriptExecutor)driver).ExecuteScript("document.getElementById(\"slideBlock\").style.opacity = 1;");
+                action.ClickAndHold(sliderButton).Build().Perform();
+                #region 开始拟人移动
+                var d1 = Math.Round(bgX * 0.8);
+                var d2 = Math.Round(bgX * 0.1);
+                var d3 = Math.Round(bgX * 0.1);
+                action.MoveByOffset((int)d1, 1);
+                Thread.Sleep(new Random().Next(100) + 150);
+                action.MoveByOffset((int)d2, 1);
+                Thread.Sleep(new Random().Next(100) + 150);
+                action.MoveByOffset((int)d3, 1);
+                action.MoveByOffset((bgX - (int)d1 - (int)d2 - (int)d3), 1);
+                Thread.Sleep(2000);
+                #endregion
+                action.Release(sliderButton);
+                action.Build().Perform();
+                driver.SwitchTo().DefaultContent();
+            }
+            return Task.CompletedTask;
+        }
+
+        private static int FireFoxGetBoundary(string fileBgPath,string fileBlockPath)
+        {
+            var dirpath = $"{Environment.CurrentDirectory}\\geetest";
+            var bgImg = Cv2.ImRead(fileBgPath);
+            var tpImg = Cv2.ImRead(fileBlockPath);
+
+            var bgEdge = new Mat();
+            Cv2.Canny(bgImg, bgEdge, 100, 200);
+      
+            var tpEdge = new Mat();
+            Cv2.Canny(tpImg, edges: tpEdge, 100, 200);
+       
+
+            var bgPic = new Mat();
+            Cv2.CvtColor(bgEdge, bgPic, ColorConversionCodes.GRAY2RGB);
+
+            var tpPic = new Mat();
+            Cv2.CvtColor(tpEdge, tpPic, ColorConversionCodes.GRAY2RGB);
+         
+
+            var res = new Mat();
+            Cv2.MatchTemplate(bgPic, tpPic, res, TemplateMatchModes.CCoeffNormed);
+            OpenCvSharp.Point minLoc = new OpenCvSharp.Point(), maxLoc = new OpenCvSharp.Point();
+            Cv2.MinMaxLoc(res, out double minVal, out double maxVal, out minLoc, out maxLoc);
+
+            var bt = new OpenCvSharp.Point(maxLoc.X + tpPic.Width, maxLoc.Y + tpPic.Height);
+            Cv2.Rectangle(bgImg, maxLoc, bt, Scalar.Red, 2);
+            string newImage = $"{dirpath}\\{DateTime.Now.Ticks}_NewBg.png";
+            Cv2.ImWrite(newImage, bgImg);
+            File.Delete(fileBgPath);
+            File.Delete(fileBlockPath);
+            File.Delete(newImage);
+            return maxLoc.X - 20;
+        }
+
+        
+
         public static Task BeginSliderLoginHandler(ref IWebDriver driver)
         {
             //等待滑块加载
             Thread.Sleep(5000);
             driver.SwitchTo().Frame(driver.FindElement(By.Id("tcaptcha_iframe")));
-            ((IJavaScriptExecutor)driver).ExecuteScript("document.getElementById(\"slideBlock\").style.opacity = 0;");
+            ((IJavaScriptExecutor)driver).ExecuteScript("document.getElementById(\"slideBlock\").style.display = none;");
             var bgPng = ((ITakesScreenshot)driver.FindElement(By.Id("slideBg"))).GetScreenshot();
             ((IJavaScriptExecutor)driver).ExecuteScript("document.getElementById(\"slideBlock\").style.opacity = 1;document.getElementById(\"slideBg\").style.opacity = 0;");
             var blockPng = ((ITakesScreenshot)driver.FindElement(By.Id("slideBlock"))).GetScreenshot();

EduSpider/Spiders/ClassInRule/TeacherManager.cs

@@ -30,11 +30,10 @@ namespace EduSpider.Spiders.ClassInRule
                 string paramStr = string.Format("page={0}&perpage={1}", pageIndex, perpage);
                 var content = new StringContent(paramStr, System.Text.Encoding.UTF8, "application/x-www-form-urlencoded");
                 var response = await request.PostAsync("https://console.eeo.cn/saasajax/teacher.ajax.php?action=getSchoolTeacherFullList", content);
-
                 var result = response.Content.ReadAsStringAsync().Result;
-
                 if (!string.IsNullOrWhiteSpace(result))
                 {
+                    VTX.FW.Helper.LogHelper.WriteInfo("RunTeacher", $"result： {result}");
                     JObject rootObj = JObject.Parse(result);
                     if (!string.IsNullOrWhiteSpace(rootObj.GetString("data")))
                     {