Merge branch 'master' of http://gitlab.oytour.com/viitto/eduspider

EduSpider.IRepository/IStudentRepository.cs

+using EduSpider.Model.Entity;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using VTX.FW.Config;
+using VTX.FW.DB;
+
+namespace EduSpider.IRepository
+{
+
+    public interface IStudentRepository: IDBRepository<RB_Student>, IDependency
+    {
+        /// <summary>
+        /// 批量操作学生资料
+        /// </summary>
+        /// <param name="students"></param>
+        /// <returns></returns>
+        public bool BatchSetStudent(List<RB_Student> students);
+    }
+}

EduSpider.Repository/StudentRepository.cs

-using System;
+using EduSpider.IRepository;
+using EduSpider.Model.Entity;
+using EduSpider.Repository.Base;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
@@ -6,7 +9,13 @@ using System.Threading.Tasks;
 
 namespace EduSpider.Repository
 {
-    public class StudentRepository 
+    public class StudentRepository : BaseRepository<RB_Student>, IStudentRepository
     {
+        public bool BatchSetStudent(List<RB_Student> students)
+        {
+            bool flag;
+            flag = base.BatchInsert(students, isReplace: true);
+            return flag;
+        }
     }
 }

EduSpider.Utility/JObjectHelper.cs

+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace EduSpider.Utility
+{
+    public static class JObjectHelper
+    {
+        public static bool TryFromString(this JObject obj, string str)
+        {
+            var flag = true;
+            try
+            {
+                obj = JObject.Parse(str);
+            }
+            catch (Exception)
+            {
+                flag = false;
+                obj=new JObject();
+            }
+
+            return flag;
+        }
+    }
+}

EduSpider.sln

 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 16
-VisualStudioVersion = 16.0.31729.503
+# Visual Studio Version 17
+VisualStudioVersion = 17.1.32210.238
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EduSpider.Model", "EduSpider.Model\EduSpider.Model.csproj", "{58B2199B-FE72-4F48-972D-F7FE32A28A0F}"
 EndProject

EduSpider/Spiders/ClassInRule/ClassInAccountManager.cs

@@ -24,7 +24,7 @@ namespace TicketSpider.Spiders.ClassInRule
 
         private static object  lockerSafe= new object();
 
-        private static string _loginCookies = "locationArgumentLang=zh-CN; sensorsdata2015jssdkcross={\"distinct_id\":\"12780052\",\"first_id\":\"180e074a47f92f-06489d0bb8915b-6b3e555b-1327104-180e074a4805d2\",\"props\":{\"$latest_traffic_source_type\":\"直接流量\",\"$latest_search_keyword\":\"未取到值_直接打开\",\"$latest_referrer\":\"\"},\"$device_id\":\"180e074a47f92f-06489d0bb8915b-6b3e555b-1327104-180e074a4805d2\"}; _eeos_uid=12780052; _eeos_useraccount=18140082327; _eeos_userlogo=/upload/images/20200126/709df9532bee722f2268_70.jpg; _eeos_domain=.eeo.cn; _eeos_sid=12780052; _eeos_nsid=kspGu2gfMu+U2EFgLpSGyg==; __tk_id=5792ca4f7a6655c40c4431fbdb729da4; PHPSESSID=l5aanmebp5lvpcbg630ccfha37; _eeos_traffic=EqlIOFq4S2hz5FTTtHzNUViBYcWCixrJYMCF6Y5yJuD8eq8ZwPvVSHMvUkLFA5lzNoaTrDBpGhEY7ykZ9lwMAsNJoNiDt2xKQBr9csRmiBE=";
+        private static string _loginCookies = null;//"locationArgumentLang=zh-CN; sensorsdata2015jssdkcross={\"distinct_id\":\"12780052\",\"first_id\":\"180e074a47f92f-06489d0bb8915b-6b3e555b-1327104-180e074a4805d2\",\"props\":{\"$latest_traffic_source_type\":\"直接流量\",\"$latest_search_keyword\":\"未取到值_直接打开\",\"$latest_referrer\":\"\"},\"$device_id\":\"180e074a47f92f-06489d0bb8915b-6b3e555b-1327104-180e074a4805d2\"}; _eeos_uid=12780052; _eeos_useraccount=18140082327; _eeos_userlogo=/upload/images/20200126/709df9532bee722f2268_70.jpg; _eeos_domain=.eeo.cn; _eeos_sid=12780052; _eeos_nsid=kspGu2gfMu+U2EFgLpSGyg==; __tk_id=5792ca4f7a6655c40c4431fbdb729da4; PHPSESSID=l5aanmebp5lvpcbg630ccfha37; _eeos_traffic=EqlIOFq4S2hz5FTTtHzNUViBYcWCixrJYMCF6Y5yJuD8eq8ZwPvVSHMvUkLFA5lzNoaTrDBpGhEY7ykZ9lwMAsNJoNiDt2xKQBr9csRmiBE=";
 
         private static bool _openHeartTimer = false;
 

EduSpider/Spiders/ClassInRule/StudentManager.cs

 using EduSpider.Model.Entity;
 using EduSpider.Repository;
-using EduSpider.Utility.Enum;
+using EduSpider.Utility;
+using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using System;
 using System.Collections.Generic;
@@ -15,15 +16,34 @@ namespace EduSpider.Spiders.ClassInRule
         public async void RunAsync(string cookie)
         {
             var request = Utility.HttpHelper.GenerateHttp(cookie);
-            var paramStr = "page=1&perpage=20&isdel=0";
-            var content = new StringContent(paramStr, System.Text.Encoding.UTF8, "application/x-www-form-urlencoded");
-            var response = await request.PostAsync("https://console.eeo.cn/saasajax/student.ajax.php?action=getSchoolStudentListByPage", content);
+            int pageIndex = 1, pageCount = 1, pageSize=10000;
 
-            var result = response.Content.ReadAsStringAsync().Result;
-        }
+            while (pageIndex <= pageCount) 
+            {
+                var paramStr = $"page={pageIndex}&perpage={pageSize}&isdel=0";
+                var content = new StringContent(paramStr, System.Text.Encoding.UTF8, "application/x-www-form-urlencoded");
+                var response = await request.PostAsync("https://console.eeo.cn/saasajax/student.ajax.php?action=getSchoolStudentListByPage", content);
+
+                var resultStr = response.Content.ReadAsStringAsync().Result ?? string.Empty;
+                var result = new JObject();
+                if (result.TryFromString(resultStr))
+                {
+                    var data = (result.ContainsKey("data") ? result["data"] : new JObject()) as JObject;
+                    if (data.ContainsKey("totalStudentNum")) {
+                        #region 更新PageCount
 
+                        if (pageIndex == 1)
+                        {
+                            pageCount = (int)Math.Ceiling(data.GetDecimal("totalStudentNum") / pageSize);
+                        }
 
+                        #endregion
+                        var students = JsonConvert.DeserializeObject<RB_Student>(data.GetString("studentList"));
+                    }
+                }
+            }
 
+        }
 
     }
 }