Merge branch 'master' of http://gitlab.oytour.com/viitto/eduspider

# Conflicts:
#	.vs/EduSpider/DesignTimeBuild/.dtbcache.v2

EduSpider.IRepository/ICourseRepository.cs

@@ -12,13 +12,13 @@ namespace EduSpider.Repository
     /// <summary>
     /// 课程仓储接口
     /// </summary>
-    public interface ICourseRepository : IDBRepository<Course>, IDependency
+    public interface ICourseRepository : IDBRepository<RB_Course>, IDependency
     {
         /// <summary>
         /// 批量新增课程
         /// </summary>
         /// <param name="courses"></param>
         /// <returns></returns>
-        public bool BatchSetCourse(List<Course> courses);
+        public bool BatchSetCourse(List<RB_Course> courses);
     }
 }

EduSpider.Model/Entity/Course.cs

@@ -8,8 +8,21 @@ namespace EduSpider.Model.Entity
     /// </summary>
     [Serializable]
     [DB(ConnectionName = "DefaultConnection")]
-    public class Course
+    public class RB_Course
     {
+        /// <summary>
+        /// 课程编号
+        /// </summary>
+        public int courseId { get; set; }
 
+        /// <summary>
+        /// 课程名称
+        /// </summary>
+        public string courseName { get; set; }
+
+        /// <summary>
+        /// 开始时间
+        /// </summary>
+        public int beginTime { get; set; }
     }
 }

EduSpider.Repository/Base/BaseRepository.cs

@@ -35,6 +35,10 @@ namespace EduSpider.Repository.Base
 
             }
             DBConnStr = VTX.FW.Helper.ConfigHelper.GetConnectionString(connKey);
+            if (string.IsNullOrWhiteSpace(DBConnStr))
+            {
+                DBConnStr = "server=192.168.10.214;user id=reborn;password=Reborn@2018;database=reborn_think;CharSet=utf8mb4; Convert Zero Datetime=true; ";
+            }
         }
 
 

EduSpider.Repository/CourseRepository.cs

@@ -11,14 +11,14 @@ namespace EduSpider.Repository
     /// <summary>
     /// 课程仓储层
     /// </summary>
-    public class CourseRepository : BaseRepository<Course>, ICourseRepository
+    public class CourseRepository : BaseRepository<RB_Course>, ICourseRepository
     {
         /// <summary>
         /// 批量新增课程
         /// </summary>
         /// <param name="courses"></param>
         /// <returns></returns>
-        public bool BatchSetCourse(List<Course> courses)
+        public bool BatchSetCourse(List<RB_Course> courses)
         {
             bool flag;
             flag = base.BatchInsert(courses, isReplace: true);

EduSpider.Utility/HttpHelper.cs

 using Newtonsoft.Json.Linq;
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Net;
+using System.Net.Http;
 using System.Text;
 using VTX.FW.Helper;
 
+
 namespace EduSpider.Utility
 {
     /// <summary>
@@ -12,126 +15,56 @@ namespace EduSpider.Utility
     /// </summary>
     public class HttpHelper
     {
-        /// <summary>
-        /// 域名
-        /// </summary>
-        private static string domain = "https://console.eeo.cn/saasajax";
-
-        /// <summary>
-        /// Cookie
-        /// </summary>
-        private static string Cookie = "sajssdk_2015_cross_new_user=1; _eeos_uid=12780052; _eeos_useraccount=18140082327; _eeos_userlogo=%2Fupload%2Fimages%2F20200126%2F709df9532bee722f2268_70.jpg; _eeos_domain=.eeo.cn; _eeos_remember=1; _eeos_traffic=EqlIOFq4S2hz5FTTtHzNUViBYcWCixrJYMCF6Y5yJuD8eq8ZwPvVSHMvUkLFA5lzNoaTrDBpGhG4CE5Tjuw%2BomVQCAXfwH%2BefAyKrRMPc3g%3D; _eeos_sid=12780052; _eeos_nsid=kspGu2gfMu%2BU2EFgLpSGyg%3D%3D; locationArgumentLang=zh-CN; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%2212780052%22%2C%22first_id%22%3A%22180f54ced3048f-02eaffc35f728a-3e604809-2073600-180f54ced316ee%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E8%87%AA%E7%84%B6%E6%90%9C%E7%B4%A2%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC%22%2C%22%24latest_referrer%22%3A%22https%3A%2F%2Fwww.baidu.com%2Flink%22%7D%2C%22%24device_id%22%3A%22180f54ced3048f-02eaffc35f728a-3e604809-2073600-180f54ced316ee%22%7D; PHPSESSID=sd33g5i5ta97b7a8d19vld71q2; __tk_id=0a89552a4b2a69f62bc9ad086c6402e1";
-
-        /// <summary>
-        /// 类型
-        /// </summary>
-        private static string contentType = "application/x-www-form-urlencoded";
-
-        /// <summary>
-        /// 获取班级信息
-        /// </summary>
-        /// <returns></returns>
-        public static string GetClassQuery()
-        {
-            string url = "/course.ajax.php?action=getCourseList";
-            string ApiUrl = domain + url;
-            string apiResult = HttpPost(ApiUrl, "", contentType:contentType, cookie: Cookie);
-            if (!string.IsNullOrWhiteSpace(apiResult))
+        public static HttpClient GenerateHttp()
         {
-                JObject rootObj = JObject.Parse(apiResult);
-                int PageSize = rootObj.GetInt("PageSize");
-                int PageCount= rootObj.GetInt("PageCount");
-            }
+            var handler = new HttpClientHandler();
+            handler.AllowAutoRedirect = false;
+            handler.UseCookies = true;
+            handler.CookieContainer = CreateCookie();
+            handler.AutomaticDecompression = DecompressionMethods.GZip;
+            handler.ClientCertificateOptions = ClientCertificateOption.Automatic;
+            var http = new HttpClient(handler);
+            GenerateHttpHeader(ref http);
 
-            return apiResult;
+            return http;
         }
 
-        public static string GetData(string url)
+        private static void GenerateHttpHeader(ref HttpClient http)
         {
-            return "";
-        }
+            http.DefaultRequestHeaders.Add("Accept", "application/json, text/plain, */*");
+            http.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br");
+            http.DefaultRequestHeaders.Add("Accept-Language", "zh-CN,zh;q=0.9,ja;q=0.8");
+            http.DefaultRequestHeaders.Add("Cache-Control", "no-cache");
+            http.DefaultRequestHeaders.Add("Origin", "https://console.eeo.cn");
+            http.DefaultRequestHeaders.Add("Pragma", "no-cache");
+            http.DefaultRequestHeaders.Add("Referer", "https://console.eeo.cn/saas/school/index.html");
+            http.DefaultRequestHeaders.Add("salesChannel", "NzcwMQ==");
+            http.DefaultRequestHeaders.Add("sec-ch-ua", "\" Not; A Brand\";v=\"99\", \"Google Chrome\";v=\"97\", \"Chromium\";v=\"97\"");
+            http.DefaultRequestHeaders.Add("sec-ch-ua-mobile", "?0");
+            http.DefaultRequestHeaders.Add("sec-ch-ua-platform", "\"Windows\"");
+            http.DefaultRequestHeaders.Add("Sec-Fetch-Dest", "empty");
+            http.DefaultRequestHeaders.Add("Sec-Fetch-Mode", "cors");
 
+            http.DefaultRequestHeaders.Add("Sec-Fetch-Site", "same-origin");
+            http.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36");
 
-        /// <summary>
-        /// Post提交数据
-        /// </summary>
-        /// <param name="url">url地址</param>
-        /// <param name="body">参数</param>
-        /// <param name="contentType">参数</param>
-        /// <returns></returns>
-        public static string HttpPost(string url, string body, string contentType = "", string cookie = "")
-        {
-            string result = "";
-            try
-            {
-                Encoding encoding = Encoding.UTF8;
-                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
-                request.Method = "POST";
 
-                request.Accept = "application/json, text/javascript, */*";
-                request.ContentType = "application/json; charset=utf-8";
-                if (contentType != "")
-                {
-                    request.ContentType = contentType;
-                }
-                if (!string.IsNullOrWhiteSpace(cookie))
-                {
-                    request.Headers.Add("cookie", cookie);
-                }
-                byte[] buffer = encoding.GetBytes(body);
-                request.ContentLength = buffer.Length;
-                request.GetRequestStream().Write(buffer, 0, buffer.Length);
-                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
-                using StreamReader reader = new StreamReader(response.GetResponseStream(), encoding);
-                result = reader.ReadToEnd();
+            http.DefaultRequestHeaders.Add("site", "zh_CN");
+            http.DefaultRequestHeaders.Add("shakehand", "535206fb27efbaf75066ff179b975ff6");
+            http.DefaultRequestHeaders.Add("pragma", "no-cache");
         }
-            catch (WebException ex)
-            {
-                LogHelper.WriteError("HttpPost", string.Format("HttpHelper_HttpPost：url:{0} body:{1} contentType:{2}", url, body, contentType), ex: ex);
-            }
-            return result;
-        }
-
 
-        /// <summary>
-        /// 执行HTTP GET请求。
-        /// </summary>
-        /// <param name="url">请求地址</param>
-        /// <returns>HTTP响应</returns>
-        public static string HttpGet(string url)
-        {
-            string result = "";
-            HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
-            req.Method = "GET";
-            req.ContentType = "application/json";
-            HttpWebResponse rsp = (HttpWebResponse)req.GetResponse();
-            Encoding encoding = Encoding.UTF8;
-            if (!string.IsNullOrWhiteSpace(rsp.CharacterSet))
-            {
-                encoding = Encoding.GetEncoding(rsp.CharacterSet);
-            }
-            Stream stream = null;
-            StreamReader reader = null;
-            try
+        private static CookieContainer CreateCookie()
         {
-                // 以字符流的方式读取HTTP响应
-                stream = rsp.GetResponseStream();
-                reader = new StreamReader(stream, encoding);
-                result = reader.ReadToEnd();
-            }
-            catch (Exception ex)
-            {
-                LogHelper.WriteError("HttpHelper_HttpGet", string.Format("url:{0}", url), ex);
-                result = "";
-            }
-            finally
+            var cookieStr = "sajssdk_2015_cross_new_user=1; PHPSESSID=2mvvd4h9hmvs9gvmv3ego8t167; _eeos_uid=12780052; _eeos_useraccount=18140082327; _eeos_userlogo=%2Fupload%2Fimages%2F20200126%2F709df9532bee722f2268_70.jpg; _eeos_domain=.eeo.cn; _eeos_remember=1; _eeos_traffic=EqlIOFq4S2hz5FTTtHzNUViBYcWCixrJYMCF6Y5yJuD8eq8ZwPvVSHMvUkLFA5lzNoaTrDBpGhFOuRmo3R6CQ4pXDRdN9nKnxel4M3FHU3U%3D; _eeos_sid=12780052; _eeos_nsid=kspGu2gfMu%2BU2EFgLpSGyg%3D%3D; locationArgumentLang=zh-CN; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%2212780052%22%2C%22first_id%22%3A%22180f59907abbed-081e4952c5b26f-12333272-2073600-180f59907acbc8%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%22180f59907abbed-081e4952c5b26f-12333272-2073600-180f59907acbc8%22%7D; __tk_id=afefff800c5a030d578e4bddba2a0914";
+            var uri = new Uri("https://console.eeo.cn");
+            var cc = new CookieContainer();
+
+            foreach (var str in cookieStr.Split(';'))
             {
-                // 释放资源
-                if (reader != null) reader.Close();
-                if (stream != null) stream.Close();
-                if (rsp != null) rsp.Close();
+                cc.SetCookies(uri, str);
             }
-            return result;
+            return cc;
         }
     }
 }
\ No newline at end of file

EduSpider.WebApi/Controllers/CourseController.cs

@@ -23,10 +23,8 @@ namespace EduSpider.WebApi.Controllers
 
         public ApiResult GetCoursePage()
         {
-            string roomApi = "api/Classroom/Query";
-            string classQuery = "api/Class/Query";
-            string str = Utility.HttpHelper.GetData(classQuery);
-            return ApiResult.Success(data: str, message: "成功!");
+            
+            return ApiResult.Success(data: "", message: "成功!");
         }
     }
 }

EduSpider.sln

@@ -23,6 +23,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EduSpider.Utility", "EduSpi
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EduSpider", "EduSpider\EduSpider.csproj", "{DAC5E8C6-98DF-4B33-9E2C-452DDE54A5FA}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ConsoleApp1", "ConsoleApp1\ConsoleApp1.csproj", "{4FB5873A-A964-4D19-AC8E-BD72A66190BF}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -61,6 +63,10 @@ Global
 		{DAC5E8C6-98DF-4B33-9E2C-452DDE54A5FA}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{DAC5E8C6-98DF-4B33-9E2C-452DDE54A5FA}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{DAC5E8C6-98DF-4B33-9E2C-452DDE54A5FA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{4FB5873A-A964-4D19-AC8E-BD72A66190BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4FB5873A-A964-4D19-AC8E-BD72A66190BF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4FB5873A-A964-4D19-AC8E-BD72A66190BF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4FB5873A-A964-4D19-AC8E-BD72A66190BF}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

EduSpider/EduSpider.csproj

@@ -31,6 +31,10 @@
 
   <ItemGroup>
     <PackageReference Include="DotnetSpider" Version="5.0.9" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="6.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="6.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="6.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="6.0.0" />
     <PackageReference Include="OpenCvSharp4.Windows" Version="4.5.5.20211231" />
     <PackageReference Include="Selenium.WebDriver" Version="4.1.0" />
     <PackageReference Include="Selenium.WebDriver.ChromeDriver" Version="100.0.4896.6000" />
@@ -38,7 +42,16 @@
     <PackageReference Include="Serilog.Sinks.Console" Version="3.1.1" />
     <PackageReference Include="Serilog.Sinks.RollingFile" Version="3.3.0" />
     <PackageReference Include="Serilog.Sinks.PeriodicBatching" Version="2.3.0" />
+    <PackageReference Include="SqlSugarCore" Version="5.0.8.3" />
+    <PackageReference Include="System.Data.SqlClient" Version="4.8.3" />
     <PackageReference Include="System.Drawing.Common" Version="6.0.0" />
+    <PackageReference Include="VTX.FW" Version="1.0.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\EduSpider.IRepository\EduSpider.IRepository.csproj" />
+    <ProjectReference Include="..\EduSpider.Model\EduSpider.Model.csproj" />
+    <ProjectReference Include="..\EduSpider.Repository\EduSpider.Repository.csproj" />
   </ItemGroup>
 
 </Project>

EduSpider/Spiders/ClassInRule/ClassInFlow.cs

@@ -22,9 +22,8 @@ namespace TicketSpider.Spiders.ClassInRule
             string loginCookies = ClassInAccountManager.GetInstance();
             Console.WriteLine("开始获取学员信息");
             new StudentManager().RunAsync();
+            new StudentManager().RunCourse();
             
         }
-
-
     }
 }

EduSpider/Spiders/ClassInRule/StudentManager.cs

 using EduSpider.Helpers;
+using EduSpider.Model.Entity;
+using EduSpider.Repository;
+using Newtonsoft.Json.Linq;
 using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Net.Http;
 using System.Text;
 using System.Threading.Tasks;
+using VTX.FW;
+using VTX.FW.Helper;
 
 namespace EduSpider.Spiders.ClassInRule
 {
@@ -12,7 +17,7 @@ namespace EduSpider.Spiders.ClassInRule
     {
         public async void RunAsync()
         {
-            var request = HttpHelper.GenerateHttp();
+            var request = EduSpider.Helpers.HttpHelper.GenerateHttp();
             var paramStr = "page=1&perpage=20&isdel=0";
             var content = new StringContent(paramStr, System.Text.Encoding.UTF8, "application/x-www-form-urlencoded");
             var response = await request.PostAsync("https://console.eeo.cn/saasajax/student.ajax.php?action=getSchoolStudentListByPage", content);
@@ -20,6 +25,45 @@ namespace EduSpider.Spiders.ClassInRule
             var result = response.Content.ReadAsStringAsync().Result;
         }
 
+        /// <summary>
+        /// 获取课程列表
+        /// </summary>
+        public async void RunCourse()
+        {
+            var request = EduSpider.Helpers.HttpHelper.GenerateHttp();
+            var paramStr = "page=1&perpage=20&courseState=1";
+            var content = new StringContent(paramStr, System.Text.Encoding.UTF8, "application/x-www-form-urlencoded");
+            var response = await request.PostAsync("https://console.eeo.cn/saasajax/course.ajax.php?action=getCourseList", content);
+
+            var result = response.Content.ReadAsStringAsync().Result;
+            List<RB_Course> list = new List<RB_Course>();
+            if (!string.IsNullOrWhiteSpace(result))
+            {
+                JObject rootObj = JObject.Parse(result);
+                if (!string.IsNullOrWhiteSpace(rootObj.GetString("data")))
+                {
+                    JObject dataObj = JObject.Parse(rootObj.GetString("data"));
+                    var totalNum = dataObj.GetInt("totalCourseNum");
+                    JArray courseArray = JArray.Parse(dataObj.GetString("courseList"));
+                    if (courseArray != null && courseArray.Count > 0)
+                    {
+                        foreach (var jItem in courseArray)
+                        {
+                            JObject courseObj = JObject.Parse(jItem.ToString());
+                            list.Add(new RB_Course()
+                            {
+                                courseId = courseObj.GetInt("courseId"),
+                                courseName=courseObj.GetString("courseName"),
+                            });
+                        }
+                    }
+                }
+            }
+            ICourseRepository courseRepository = new CourseRepository();
+            courseRepository.BatchSetCourse(list);
+            Console.WriteLine("result", result);
+        }
+
 
     }
 }