新增测验抓取

.gitignore

@@ -3,3 +3,4 @@ obj/
 .vs/
 EduSpider.WebApi/EduSpider.WebApi.csproj.user
 EduSpider.WebApi/Properties/PublishProfiles/FolderProfile.pubxml.user
+EduSpider/EduSpider.csproj.user

EduSpider.Model/Entity/RB_Stu_Comment.cs

@@ -18,12 +18,12 @@ namespace EduSpider.Model.Entity
         /// <summary>
         /// 开始作业编号
         /// </summary>
-        public int StartHomeWorkId { get; set; }
+        public Int64 StartHomeWorkId { get; set; }
 
         /// <summary>
         /// 家庭作业编号
         /// </summary>
-        public int HomeWorkId { get; set; }
+        public Int64 HomeWorkId { get; set; }
 
         /// <summary>
         /// 学员登录编号

EduSpider.Model/Entity/RB_Stu_HomeWork.cs

@@ -13,7 +13,7 @@ namespace EduSpider.Model.Entity
         /// <summary>
         /// 学生家庭作业主键编号
         /// </summary>
-        public int stu_homework_id { get; set; }
+        public Int64 stu_homework_id { get; set; }
 
         /// <summary>
         /// admire

EduSpider.Model/Extend/ExamWorkResult.cs

@@ -19,7 +19,7 @@ namespace EduSpider.Model.Extend
         /// <summary>
         /// 作业编号
         /// </summary>
-        public int Stu_HomeWork_Id { get; set; }
+        public Int64 Stu_HomeWork_Id { get; set; }
 
         /// <summary>
         /// 创建时间

EduSpider.Model/Query/CourseQuery.cs

-namespace EduSpider.Model.Query
+using System;
+
+namespace EduSpider.Model.Query
 {
     /// <summary>
     /// 课程查询实体类
@@ -43,17 +45,17 @@
         /// <summary>
         /// 作业编号
         /// </summary>
-        public int HomeWorkId { get; set; }
+        public Int64 HomeWorkId { get; set; }
 
         /// <summary>
         /// 开始作业编号
         /// </summary>
-        public int StartHomeWorkId { get; set; }
+        public Int64 StartHomeWorkId { get; set; }
 
         /// <summary>
         /// 结束作业编号
         /// </summary>
-        public int EndHomeWorkId { get; set; }
+        public Int64 EndHomeWorkId { get; set; }
 
         /// <summary>
         /// 创建类型(1-系统创建，2-老师创建)

EduSpider.Repository/StuHomeWorkRepository.cs

@@ -437,7 +437,7 @@ WHERE 1=1
         /// <param name="CourseId"></param>
         /// <param name="Stu_HomeWork_Id"></param>
         /// <returns></returns>
-        private static RB_Stu_Comment GetSystemCreateComment(List<RB_Stu_Comment> commentList,int CourseId,int Stu_HomeWork_Id)
+        private static RB_Stu_Comment GetSystemCreateComment(List<RB_Stu_Comment> commentList,int CourseId,Int64 Stu_HomeWork_Id)
         {
             var tempModel = commentList.Where(qitem => qitem.CourseId == CourseId && qitem.HomeWorkId == Stu_HomeWork_Id && qitem.CreateType == 1).FirstOrDefault();
             return tempModel;

EduSpider.Utility/HttpHelper.cs

@@ -25,7 +25,8 @@ namespace EduSpider.Utility
                 UseCookies = true,
                 CookieContainer = CreateCookie(cookie),
                 AutomaticDecompression = DecompressionMethods.GZip,
-                ClientCertificateOptions = ClientCertificateOption.Automatic
+                ClientCertificateOptions = ClientCertificateOption.Automatic,
+                SslProtocols = System.Security.Authentication.SslProtocols.Tls
             };
             var http = new HttpClient(handler);
             GenerateHttpHeader(ref http);

EduSpider/EduSpider.csproj

@@ -14,11 +14,6 @@
 
   <ItemGroup>
     <Compile Remove="Spiders\CityService.cs" />
-    <Compile Remove="Spiders\EasterLineRule\AccountManagerHelper.cs" />
-    <Compile Remove="Spiders\EasterLineRule\CookiesHelper.cs" />
-    <Compile Remove="Spiders\EasterLineRule\PassengerHelper.cs" />
-    <Compile Remove="Spiders\EasterLineRule\QueryFlightHelper.cs" />
-    <Compile Remove="Spiders\EasterLineRule\TicketOrderHelper.cs" />
   </ItemGroup>
 
   <ItemGroup>
@@ -30,11 +25,6 @@
     <Content Include="appsettings.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </Content>
-    <Content Include="Spiders\EasterLineRule\AccountManagerHelper.cs" />
-    <Content Include="Spiders\EasterLineRule\CookiesHelper.cs" />
-    <Content Include="Spiders\EasterLineRule\PassengerHelper.cs" />
-    <Content Include="Spiders\EasterLineRule\QueryFlightHelper.cs" />
-    <Content Include="Spiders\EasterLineRule\TicketOrderHelper.cs" />
 
   </ItemGroup>
 

EduSpider/Spiders/ClassInRule/ClassInFlow.cs

@@ -46,6 +46,11 @@ namespace TicketSpider.Spiders.ClassInRule
             Console.WriteLine("作业信息更新结束");
             VTX.FW.Helper.LogHelper.WriteInfo("StartAsync", "作业信息更新结束");
 
+            Console.WriteLine("开始获取测验信息");
+            VTX.FW.Helper.LogHelper.WriteInfo("StartAsync", "开始测验信息");
+            QuestionManager.RunHomeWork(loginCookies);
+            Console.WriteLine("测验信息更新结束");
+            VTX.FW.Helper.LogHelper.WriteInfo("StartAsync", "测验信息更新结束");
 
         }
 

EduSpider/Spiders/ClassInRule/QuestionManager.cs

+using EduSpider.IRepository;
+using EduSpider.Model.Entity;
+using EduSpider.Repository;
+using EduSpider.Utility;
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net.Http;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using VTX.FW.Helper;
+
+namespace EduSpider.Spiders.ClassInRule
+{
+    public class QuestionManager
+    {
+        /// <summary>
+        /// 获取学生作业列表
+        /// </summary>
+        public static void RunHomeWork(string cookie)
+        {
+            int perpage = 200;
+            var pageCount = 1;
+            var range = $"[1609430400,{ConvertHelper.DateTimeToUnix(DateTime.Now)}]";
+            var request = Utility.HttpHelper.GenerateHttp(cookie);
+            string queryStr = "page={0}&perpage={1}&examStatus=&paperName=&courseName=&arrangeTime={2}&display=0";
+            string paramStr = string.Format(queryStr, pageCount, perpage,range);
+            var content = new StringContent(paramStr, System.Text.Encoding.UTF8, "application/x-www-form-urlencoded");
+            IHomeWorkRepository homeWorkRepository = new HomeWorkRepository();
+            IStuHomeWorkRepository stuHomeWorkRepository = new StuHomeWorkRepository();
+            int totalCount = 0;
+            Console.WriteLine(string.Format("开始第{0}次获取测验", pageCount));
+            VTX.FW.Helper.LogHelper.WriteInfo("RunHomeWork", string.Format("开始第{0}次获取测验", pageCount));
+            string url = "https://console.eeo.cn/saasajax/exam.ajax.php?action=getExamManageList";
+            var response = request.PostAsync(url, content);
+            var result = response.Result.Content.ReadAsStringAsync().Result;
+            if (!string.IsNullOrWhiteSpace(result))
+            {
+                JObject rootObj = JObject.Parse(result);
+                if (!string.IsNullOrWhiteSpace(rootObj.GetString("data")))
+                {
+                    JObject dataObj = JObject.Parse(rootObj.GetString("data"));
+                    var totalNum = dataObj.GetInt("totalSum");
+                    var tempList = ParseJson(dataObj.GetString("examsData"),cookie);
+                    if (tempList != null && tempList.Count > 0)
+                    {
+                        homeWorkRepository.BatchSetHomeWorkRepository(tempList);
+                        totalCount += tempList.Count;
+                        Console.WriteLine(string.Format("第{0}次,导入完成{1}条.", pageCount, totalCount));
+                        VTX.FW.Helper.LogHelper.WriteInfo("RunHomeWork", string.Format("第{0}次,导入完成{1}条.", pageCount, totalCount));
+                        foreach (var item in tempList)
+                        {
+                            RunStuHomeWork(cookie, item.Homework_id, item.Course_id,item);
+                        }
+                    }
+                    if (totalNum % perpage == 0)
+                    {
+                        pageCount = totalNum / perpage;
+                    }
+                    else
+                    {
+                        pageCount = totalNum / perpage + 1;
+                    }
+
+                    if (totalNum > perpage)
+                    {
+                        for (var i = 2; i <= pageCount; i++)
+                        {
+                            Thread.Sleep(1000 * 2);
+                            Console.WriteLine(string.Format("第{0}/{1}次获取课程数据", i, pageCount));
+                            VTX.FW.Helper.LogHelper.WriteInfo("RunHomeWork", string.Format("第{0}/{1}次获取课程数据", i, pageCount));
+                            paramStr = string.Format(queryStr, i, perpage, range);
+                            var sub_content = new StringContent(paramStr, System.Text.Encoding.UTF8, "application/x-www-form-urlencoded");
+                            var sub_response = request.PostAsync(url, sub_content);
+
+                            var sub_result = sub_response.Result.Content.ReadAsStringAsync().Result;
+                            if (!string.IsNullOrWhiteSpace(sub_result))
+                            {
+                                JObject sub_rootObj = JObject.Parse(sub_result);
+                                if (!string.IsNullOrWhiteSpace(sub_rootObj.GetString("data")))
+                                {
+                                    JObject sub_dataObj = JObject.Parse(sub_rootObj.GetString("data"));
+                                    var subtempList = ParseJson(sub_dataObj.GetString("list"),cookie);
+                                    if (subtempList != null && subtempList.Count > 0)
+                                    {
+                                        homeWorkRepository.BatchSetHomeWorkRepository(subtempList);
+                                        totalCount += subtempList.Count;
+                                        Console.WriteLine(string.Format("第{0}次,导入完成{1}条.", i, totalCount));
+                                        VTX.FW.Helper.LogHelper.WriteInfo("RunHomeWork", string.Format("第{0}次,导入完成{1}条.", i, totalCount));
+                                        foreach (var sItem in subtempList)
+                                        {
+                                            RunStuHomeWork(cookie, sItem.Homework_id, sItem.Course_id, sItem);
+                                        }
+                                        if (i == totalCount)
+                                        {
+                                            // stuHomeWorkRepository.CreateHomeWorkCommentRepository(0);
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        public static void RequestExamDetail(string cookie,int courseId, int examId, int tid,ref RB_HomeWork m)
+        {
+            var request = Utility.HttpHelper.GenerateHttp(cookie);
+
+            VTX.FW.Helper.LogHelper.WriteInfo("RunAsync", $"正在查询测验详细信息");
+            var paramStr = $"courseId={courseId}&examsId=[{examId}]&teachersId=[{tid}]&page=1&perpage=20";
+            var content = new StringContent(paramStr, System.Text.Encoding.UTF8, "application/x-www-form-urlencoded");
+            var response = request.PostAsync("https://console.eeo.cn/saasajax/exam.ajax.php?action=getCourseExamAnalysisList", content);
+
+            var resultStr = response.Result.Content.ReadAsStringAsync().Result ?? string.Empty;
+            var result = resultStr.TryToJObject();
+
+            if (result.flag)
+            {
+                var data = (result.val.ContainsKey("data") ? result.val["data"] : new JObject()) as JObject;
+                var homeObj = (data["examsData"] as JArray)?.FirstOrDefault() as JObject;
+                m.Num = homeObj.GetInt("shouldStSum");
+                m.Cream_num = homeObj.GetInt("excellentStSum");
+                m.Revised_num = 0; //未匹配数据;
+                m.Rnum = homeObj.GetInt("beReviewStSum");
+                m.Cnum = homeObj.GetInt("endStSum");
+
+                m.Score_value = homeObj.GetDecimal("score_value");
+
+                m.Th_cancel = string.Empty;
+
+                m.Ref_num = homeObj.GetInt("needReStSum");
+                m.Refc_num = homeObj.GetInt("alreadyReStSum");
+                m.Av_score = homeObj.GetDecimal("averageScore")/100;
+                m.Max_score = homeObj.GetDecimal("highestScore")/100;
+                m.Min_score = homeObj.GetDecimal("lowestScore")/100;
+            }
+        }
+
+        /// <summary>
+        /// JSON字符串转课程列表
+        /// </summary>
+        /// <param name="data"></param>
+        /// <returns></returns>
+        public static List<RB_HomeWork> ParseJson(string data,string cookie)
+        {
+            List<RB_HomeWork> list = new();
+            if (!string.IsNullOrWhiteSpace(data))
+            {
+                JArray courseArray = JArray.Parse(data);
+                if (courseArray != null && courseArray.Count > 0)
+                {
+                    foreach (var jItem in courseArray)
+                    {
+                        JObject homeObj = JObject.Parse(jItem.ToString());
+                        var m = new RB_HomeWork()
+                        {
+                            Homework_id = homeObj.GetInt("examId"),
+                            Homework_title = homeObj.GetString("paperName"),
+                            Homework_share_key = homeObj.GetString("examId"),
+                            Course_id = homeObj.GetInt("courseId"),
+                            Course_name = homeObj.GetString("courseName"),
+                            Start_time = ConvertHelper.UnixToDateTime(homeObj.GetInt("startTime")),
+                            End_time = ConvertHelper.UnixToDateTime(homeObj.GetInt("endTime")),
+                            Status = homeObj.GetInt("examStatus"),
+                            Is_del = 0,// 未找到隐射字段 //homeObj.GetInt("is_del"),
+                            Is_open = 1,// 未找到隐射字段 homeObj.GetInt("is_open"),
+                            Open_type = 1,// 未找到隐射字段 .GetInt("open_type"),
+                            Is_revise = 1,// 未找到隐射字段 homeObj.GetInt("is_revise"),
+                            Num = homeObj.GetInt("shouldStSum"),
+                            Cream_num = homeObj.GetInt("endStSum"),
+                            Revised_num = homeObj.GetInt("beReviewStSum"),
+                            Rnum = homeObj.GetInt("rnum"),
+                            Cnum = homeObj.GetInt("cnum"),
+                            School_uid = homeObj.GetInt("schoolId"),
+                            Score_type = 5,// homeObj.GetInt("score_type"),
+                            Score_value = homeObj.GetDecimal("score_value"),
+                            Teacher_account = homeObj.GetString("teacherAccount"),
+                            Teacher_name = homeObj.GetString("teacherName"),
+                            Teacher_uid = homeObj.GetInt("teacherId"),
+                            Th_cancel = homeObj.GetString("th_cancel"),
+                            Add_time = ConvertHelper.UnixToDateTime(homeObj.GetInt("arrangeTime")),
+                            Update_time = ConvertHelper.UnixToDateTime(homeObj.GetInt("arrangeTime")),
+                            Ref_num = homeObj.GetInt("ref_num"),
+                            Refc_num = homeObj.GetInt("refc_num"),
+                            Av_score = homeObj.GetDecimal("av_score"),
+                            Max_score = homeObj.GetDecimal("max_score"),
+                            Min_score = homeObj.GetDecimal("min_score"),
+                        };
+                        RequestExamDetail(cookie, m.Course_id, m.Homework_id, m.Teacher_uid, ref m);
+                        list.Add(m);
+                    }
+                }
+            }
+            return list;
+        }
+
+        /// <summary>
+        /// 获取学生提交作业数据
+        /// </summary>
+        /// <param name="cookie"></param>
+        /// <param name="homework_id"></param>
+        public static void RunStuHomeWork(string cookie, int homework_id, int courseId, RB_HomeWork m)
+        {
+            var request = Utility.HttpHelper.GenerateHttp(cookie);
+            IStuHomeWorkRepository stuHomeWorkRepository = new StuHomeWorkRepository();
+            ICourseStudentRepository courseStudentRepository = new CourseStudentRepository();
+            string stuIds = "";
+            var courseStuList = courseStudentRepository.GetCourseStudentListRepository(new Model.Query.CourseQuery() { CourseId = courseId });
+            if (courseStuList != null)
+            {
+                stuIds = string.Join(",", courseStuList.Select(qitem => qitem.uid));
+            }
+            string queryStr = "page=1&perpage=200&examsId=[{0}]&courseId={1}&studentsId=[{2}]";
+
+            string paramStr = string.Format(queryStr, homework_id, courseId, stuIds);
+            var content = new StringContent(paramStr, System.Text.Encoding.UTF8, "application/x-www-form-urlencoded");
+            Console.WriteLine(string.Format("开始获取{0}作业学员数据", homework_id));
+            VTX.FW.Helper.LogHelper.WriteInfo("RunStuHomeWork", string.Format("开始获取{0}作业学员数据", homework_id));
+            string url = "https://console.eeo.cn/saasajax/exam.ajax.php?action=getCourseExamsStDetailList";
+            var response = request.PostAsync(url, content);
+            var sutHomeWorkResult = response.Result.Content.ReadAsStringAsync().Result;
+            List<RB_Stu_HomeWork> list = new();
+            if (!string.IsNullOrWhiteSpace(sutHomeWorkResult))
+            {
+                JObject stuHomeRootObj = JObject.Parse(sutHomeWorkResult);
+                if (!string.IsNullOrWhiteSpace(stuHomeRootObj.GetString("data")))
+                {
+                    JObject stuHomeDataObj = JObject.Parse(stuHomeRootObj.GetString("data"));
+                    if (!string.IsNullOrWhiteSpace(stuHomeDataObj.GetString("examsData")))
+                    {
+                        JArray stuHomeArray = JArray.Parse(stuHomeDataObj.GetString("examsData"));
+                        if (stuHomeArray != null && stuHomeArray.Count > 0)
+                        {
+                            foreach (var rootItem in stuHomeArray)
+                            {
+                                JObject stuItemObj = JObject.Parse(rootItem.ToString());
+                                if (!string.IsNullOrWhiteSpace(stuItemObj.GetString("studentsDetail")))
+                                {
+                                    JArray stuListArray = JArray.Parse(stuItemObj.GetString("studentsDetail"));
+                                    if (stuListArray != null && stuListArray.Count > 0)
+                                    {
+                                        foreach (var sItem in stuListArray)
+                                        {
+                                            JObject lastObj = JObject.Parse(sItem.ToString());
+                                            var tempCream = lastObj.GetString("scoreRate").Replace("%", "");
+                                            tempCream = tempCream == "-" ? "0" : tempCream;
+                                            list.Add(new RB_Stu_HomeWork()
+                                            {
+                                                stu_homework_id = Int64.Parse($"{homework_id}{lastObj.GetInt("studentId")}"),
+                                                admire = string.Empty,
+                                                cancel = string.Empty,
+                                                correct = lastObj.GetDecimal("correctSum"),
+                                                wrong = lastObj.GetInt("mistakeSum"),
+                                                course_id = courseId,
+                                                is_cream = Convert.ToDecimal(tempCream)>=80?1:0,
+                                                emend = 0,
+                                                homework_id = homework_id,
+                                                homework_share_key = string.Empty,
+                                                is_draft = 0,
+                                                is_reform = 0,
+                                                is_revised = 1,
+                                                is_startd = 1,
+                                                readover = string.Empty,
+                                                ref_time = ConvertHelper.UnixToDateTime(lastObj.GetInt("endTime")),
+                                                comment = string.Empty,
+                                                reform = string.Empty,
+                                                school_uid = 0,
+                                                score = lastObj.GetDecimal("examScore")/100,
+                                                score_p = Convert.ToDecimal(tempCream),
+                                                score_type = 5,
+                                                show_time = ConvertHelper.UnixToDateTime(lastObj.GetInt("endTime")),
+                                                status = lastObj.GetInt("stStatus"),
+                                                stu_homework_share_key = string.Empty,
+                                                student_account = lastObj.GetString("studentAccount"),
+                                                student_name = lastObj.GetString("studentName"),
+                                                student_uid = lastObj.GetInt("studentId"),
+                                                th_account = m.Teacher_account,
+                                                th_cancel = String.Empty,
+                                                th_name = m.Teacher_name,
+                                                th_num = 1,
+                                                th_time = DateTime.Now,
+                                                th_uid = m.Teacher_uid,
+                                                add_time = ConvertHelper.UnixToDateTime(lastObj.GetInt("endTime")),
+                                                update_time = ConvertHelper.UnixToDateTime(lastObj.GetInt("endTime")),
+                                            });
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            if (list != null && list.Count > 0)
+            {
+                stuHomeWorkRepository.BatchSetStuHomeWorkRepository(list);
+            }
+        }
+    }
+}