页面修改

EduSpider/Spiders/ClassInRule/CourseManager.cs

@@ -7,6 +7,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Net.Http;
 using System.Text;
+using System.Threading;
 using VTX.FW.Helper;
 
 namespace EduSpider.Spiders.ClassInRule
@@ -22,13 +23,18 @@ namespace EduSpider.Spiders.ClassInRule
         public async void RunCourse(string cookie)
         {
             int perpage = 20;
+            var pageCount = 1;
             var request = Utility.HttpHelper.GenerateHttp(cookie);
-            string paramStr = string.Format("page={0}&perpage={1}&courseState=1", "1", perpage);
+            string queryStr = "page={0}&perpage={1}";
+            string paramStr = string.Format(queryStr, pageCount, perpage);
             var content = new StringContent(paramStr, System.Text.Encoding.UTF8, "application/x-www-form-urlencoded");
+            Console.WriteLine(string.Format("开始第{0}次获取课程数据", pageCount));
+            int totalCount = 0;
             var response = await request.PostAsync("https://console.eeo.cn/saasajax/course.ajax.php?action=getCourseList", content);
 
             var result = response.Content.ReadAsStringAsync().Result;
-            List<RB_Course> list = new();
+
+            ICourseRepository courseRepository = new CourseRepository();
             if (!string.IsNullOrWhiteSpace(result))
             {
                 JObject rootObj = JObject.Parse(result);
@@ -36,37 +42,75 @@ namespace EduSpider.Spiders.ClassInRule
                 {
                     JObject dataObj = JObject.Parse(rootObj.GetString("data"));
                     var totalNum = dataObj.GetInt("totalCourseNum");
-                    var pageCount = 1;
+                    var tempList = ParseJson(dataObj.GetString("courseList"));
+                    if (tempList != null && tempList.Count > 0)
+                    {
+                        courseRepository.BatchSetCourse(tempList);
+                        totalCount += tempList.Count();
+                        Console.WriteLine(string.Format("第{0}次,导入完成{1}条.", pageCount,totalCount));
+                    }
+                   
                     if (totalNum % perpage == 0)
                     {
-                        pageCount = totalNum % perpage;
+                        pageCount = totalNum / perpage;
                     }
                     else
                     {
-                        pageCount = totalNum % perpage + 1;
+                        pageCount = totalNum / perpage + 1;
                     }
                     if (totalNum > perpage)
                     {
+                        for (var i = 2; i <= pageCount; i++)
+                        {
+                            Thread.Sleep(1000 * 2);
+                            Console.WriteLine(string.Format("第{0}/{1}次获取课程数据", i, pageCount));
+                            paramStr = string.Format(queryStr, i, perpage);
+                            var sub_content = new StringContent(paramStr, System.Text.Encoding.UTF8, "application/x-www-form-urlencoded");
+                            var sub_response = await request.PostAsync("https://console.eeo.cn/saasajax/course.ajax.php?action=getCourseList", sub_content);
 
+                            var sub_result = sub_response.Content.ReadAsStringAsync().Result;
+                            if (!string.IsNullOrWhiteSpace(sub_result))
+                            {
+                                JObject sub_rootObj = JObject.Parse(sub_result);
+                                if (!string.IsNullOrWhiteSpace(sub_rootObj.GetString("data")))
+                                {
+                                    JObject sub_dataObj = JObject.Parse(sub_rootObj.GetString("data"));
+                                    var subtempList = ParseJson(sub_dataObj.GetString("courseList"));
+                                    if (subtempList != null && subtempList.Count > 0)
+                                    {
+                                        courseRepository.BatchSetCourse(subtempList);
+                                        totalCount += subtempList.Count();
+                                        Console.WriteLine(string.Format("第{0}次,导入完成{1}条.", i, totalCount));
+                                    }
+                                }
+                            }
+                        }
                     }
-                   
                 }
             }
-            ICourseRepository courseRepository = new CourseRepository();
-            courseRepository.BatchSetCourse(list);
+
             Console.WriteLine("result", result);
         }
 
+        /// <summary>
+        /// JSON字符串转课程列表
+        /// </summary>
+        /// <param name="data"></param>
+        /// <returns></returns>
         public List<RB_Course> ParseJson(string data)
         {
-            //dataObj.GetString("courseList")
             List<RB_Course> list = new List<RB_Course>();
-           JArray courseArray = JArray.Parse(data);
+            JArray courseArray = JArray.Parse(data);
             if (courseArray != null && courseArray.Count > 0)
             {
                 foreach (var jItem in courseArray)
                 {
                     JObject courseObj = JObject.Parse(jItem.ToString());
+                    var expiryTime = ConvertHelper.UnixToDateTime(courseObj.GetInt("expiryTime"));
+                    if (ConvertHelper.FormatTime(expiryTime) == "1970-01-01 08:00:00")
+                    {
+                        expiryTime = new DateTime(0001, 01, 01);
+                    }
                     list.Add(new RB_Course()
                     {
                         courseId = courseObj.GetInt("courseId"),
@@ -74,7 +118,7 @@ namespace EduSpider.Spiders.ClassInRule
                         beginTime = ConvertHelper.UnixToDateTime(courseObj.GetInt("beginTime")),
                         completeNum = courseObj.GetInt("completeNum"),
                         totalClassNum = courseObj.GetInt("totalClassNum"),
-                        expiryTime = ConvertHelper.UnixToDateTime(courseObj.GetInt("expiryTime")),
+                        expiryTime = expiryTime,
                         liveNum = courseObj.GetInt("liveNum"),
                         recordNum = courseObj.GetInt("recordNum"),
                         openNum = courseObj.GetInt("openNum"),
@@ -92,7 +136,6 @@ namespace EduSpider.Spiders.ClassInRule
                     }); ;
                 }
             }
-
             return list;
         }
     }