Skip to content

E
EduSpider
Commit 42cbeaab authored by 罗超's avatar 罗超
Browse files
Options

Merge branch 'master' of http://gitlab.oytour.com/viitto/eduspider

parents 618000df 872484ac
Hide whitespace changes
Inline Side-by-side
Showing with 49 additions and 109 deletions
EduSpider.Model/Entity/Course.cs
View file @ 42cbeaab
......@@ -24,5 +24,7 @@ namespace EduSpider.Model.Entity
/// 开始时间
/// </summary>
public int beginTime { get; set; }
}
}
EduSpider.Utility/HttpHelper.cs
View file @ 42cbeaab
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.IO;
using System;
using System.Net;
using System.Net.Http;
using System.Text;
using VTX.FW.Helper;
namespace EduSpider.Utility
......@@ -15,20 +10,31 @@ namespace EduSpider.Utility
/// </summary>
public class HttpHelper
{
public static HttpClient GenerateHttp()
/// <summary>
/// 创建HttpClient
/// </summary>
/// <param name="cookie"></param>
/// <returns></returns>
public static HttpClient GenerateHttp(string cookie)
{
var handler = new HttpClientHandler();
handler.AllowAutoRedirect = false;
handler.UseCookies = true;
handler.CookieContainer = CreateCookie();
handler.AutomaticDecompression = DecompressionMethods.GZip;
handler.ClientCertificateOptions = ClientCertificateOption.Automatic;
var handler = new HttpClientHandler
{
AllowAutoRedirect = false,
UseCookies = true,
CookieContainer = CreateCookie(cookie),
AutomaticDecompression = DecompressionMethods.GZip,
ClientCertificateOptions = ClientCertificateOption.Automatic
};
var http = new HttpClient(handler);
GenerateHttpHeader(ref http);
return http;
}
/// <summary>
/// 拼接Header参数
/// </summary>
/// <param name="http"></param>
private static void GenerateHttpHeader(ref HttpClient http)
{
http.DefaultRequestHeaders.Add("Accept", "application/json, text/plain, */*");
......@@ -54,9 +60,13 @@ namespace EduSpider.Utility
http.DefaultRequestHeaders.Add("pragma", "no-cache");
}
private static CookieContainer CreateCookie()
/// <summary>
/// 拼接Cookie
/// </summary>
/// <param name="cookieStr"></param>
/// <returns></returns>
private static CookieContainer CreateCookie(string cookieStr)
{
var cookieStr = "sajssdk_2015_cross_new_user=1; PHPSESSID=2mvvd4h9hmvs9gvmv3ego8t167; _eeos_uid=12780052; _eeos_useraccount=18140082327; _eeos_userlogo=%2Fupload%2Fimages%2F20200126%2F709df9532bee722f2268_70.jpg; _eeos_domain=.eeo.cn; _eeos_remember=1; _eeos_traffic=EqlIOFq4S2hz5FTTtHzNUViBYcWCixrJYMCF6Y5yJuD8eq8ZwPvVSHMvUkLFA5lzNoaTrDBpGhFOuRmo3R6CQ4pXDRdN9nKnxel4M3FHU3U%3D; _eeos_sid=12780052; _eeos_nsid=kspGu2gfMu%2BU2EFgLpSGyg%3D%3D; locationArgumentLang=zh-CN; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%2212780052%22%2C%22first_id%22%3A%22180f59907abbed-081e4952c5b26f-12333272-2073600-180f59907acbc8%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%22180f59907abbed-081e4952c5b26f-12333272-2073600-180f59907acbc8%22%7D; __tk_id=afefff800c5a030d578e4bddba2a0914";
var uri = new Uri("https://console.eeo.cn");
var cc = new CookieContainer();
......
EduSpider/EduSpider.csproj
View file @ 42cbeaab
......@@ -52,6 +52,7 @@
<ProjectReference Include="..\EduSpider.IRepository\EduSpider.IRepository.csproj" />
<ProjectReference Include="..\EduSpider.Model\EduSpider.Model.csproj" />
<ProjectReference Include="..\EduSpider.Repository\EduSpider.Repository.csproj" />
<ProjectReference Include="..\EduSpider.Utility\EduSpider.Utility.csproj" />
</ItemGroup>
</Project>
EduSpider/Helpers/HttpHelper.cs deleted 100644 → 0
View file @ 618000df
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
using TicketSpider.Spiders.ClassInRule;
namespace EduSpider.Helpers
{
public class HttpHelper
{
public static HttpClient GenerateHttp()
{
var handler = new HttpClientHandler();
handler.AllowAutoRedirect = false;
handler.UseCookies = true;
handler.CookieContainer = CreateCookie();
handler.AutomaticDecompression = DecompressionMethods.GZip;
handler.ClientCertificateOptions = ClientCertificateOption.Automatic;
var http = new HttpClient(handler);
GenerateHttpHeader(ref http);
return http;
}
private static void GenerateHttpHeader(ref HttpClient http)
{
http.DefaultRequestHeaders.Add("Accept", "application/json, text/plain, */*");
http.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br");
http.DefaultRequestHeaders.Add("Accept-Language", "zh-CN,zh;q=0.9,ja;q=0.8");
http.DefaultRequestHeaders.Add("Cache-Control", "no-cache");
http.DefaultRequestHeaders.Add("Origin", "https://console.eeo.cn");
http.DefaultRequestHeaders.Add("Pragma", "no-cache");
http.DefaultRequestHeaders.Add("Referer", "https://console.eeo.cn/saas/school/index.html");
http.DefaultRequestHeaders.Add("salesChannel", "NzcwMQ==");
http.DefaultRequestHeaders.Add("sec-ch-ua", "\" Not; A Brand\";v=\"99\", \"Google Chrome\";v=\"97\", \"Chromium\";v=\"97\"");
http.DefaultRequestHeaders.Add("sec-ch-ua-mobile", "?0");
http.DefaultRequestHeaders.Add("sec-ch-ua-platform", "\"Windows\"");
http.DefaultRequestHeaders.Add("Sec-Fetch-Dest", "empty");
http.DefaultRequestHeaders.Add("Sec-Fetch-Mode", "cors");
http.DefaultRequestHeaders.Add("Sec-Fetch-Site", "same-origin");
http.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36");
http.DefaultRequestHeaders.Add("site", "zh_CN");
http.DefaultRequestHeaders.Add("shakehand", "535206fb27efbaf75066ff179b975ff6");
http.DefaultRequestHeaders.Add("pragma", "no-cache");
}
private static CookieContainer CreateCookie()
{
var cookieStr = ClassInAccountManager.GetInstance();
var uri = new Uri("https://console.eeo.cn");
var cc = new CookieContainer();
foreach (var str in cookieStr.Split(';'))
{
cc.SetCookies(uri, str);
}
return cc;
}
}
}
EduSpider/Program.cs
View file @ 42cbeaab
// See https://aka.ms/new-console-template for more information
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using TicketSpider.Spiders;
using TicketSpider.Spiders.ClassInRule;
public class Program
namespace EduSpider
{
static void Main(string[] args)
public class Program
{
static void Main(string[] args)
{
ThreadPool.SetMaxThreads(255, 255);
ThreadPool.SetMinThreads(255, 255);
ThreadPool.SetMaxThreads(255, 255);
ThreadPool.SetMinThreads(255, 255);
new ClassInFlow().StartAsync();
new ClassInFlow().StartAsync();
Console.ReadLine();
Console.ReadLine();
Environment.Exit(0);
Environment.Exit(0);
}
}
}
\ No newline at end of file
EduSpider/Spiders/ClassInRule/ClassInFlow.cs
View file @ 42cbeaab
......@@ -15,14 +15,13 @@ namespace TicketSpider.Spiders.ClassInRule
public class ClassInFlow
{
public async void StartAsync()
{
Console.WriteLine("开始模拟登录信息.......");
string loginCookies = ClassInAccountManager.GetInstance();
Console.WriteLine("开始获取学员信息");
new StudentManager().RunAsync();
new StudentManager().RunCourse();
new StudentManager().RunAsync(loginCookies);
new StudentManager().RunCourse(loginCookies);
}
}
......
EduSpider/Spiders/ClassInRule/StudentManager.cs
View file @ 42cbeaab
using EduSpider.Helpers;
using EduSpider.Model.Entity;
using EduSpider.Model.Entity;
using EduSpider.Repository;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
using VTX.FW;
using VTX.FW.Helper;
namespace EduSpider.Spiders.ClassInRule
{
public class StudentManager
{
public async void RunAsync()
public async void RunAsync(string cookie)
{
var request = EduSpider.Helpers.HttpHelper.GenerateHttp();
var request = Utility.HttpHelper.GenerateHttp(cookie);
var paramStr = "page=1&perpage=20&isdel=0";
var content = new StringContent(paramStr, System.Text.Encoding.UTF8, "application/x-www-form-urlencoded");
var response = await request.PostAsync("https://console.eeo.cn/saasajax/student.ajax.php?action=getSchoolStudentListByPage", content);
......@@ -28,15 +24,15 @@ namespace EduSpider.Spiders.ClassInRule
/// <summary>
/// 获取课程列表
/// </summary>
public async void RunCourse()
public async void RunCourse(string cookie)
{
var request = EduSpider.Helpers.HttpHelper.GenerateHttp();
var request = Utility.HttpHelper.GenerateHttp(cookie);
var paramStr = "page=1&perpage=20&courseState=1";
var content = new StringContent(paramStr, System.Text.Encoding.UTF8, "application/x-www-form-urlencoded");
var response = await request.PostAsync("https://console.eeo.cn/saasajax/course.ajax.php?action=getCourseList", content);
var result = response.Content.ReadAsStringAsync().Result;
List<RB_Course> list = new List<RB_Course>();
List<RB_Course> list = new();
if (!string.IsNullOrWhiteSpace(result))
{
JObject rootObj = JObject.Parse(result);
......@@ -54,6 +50,8 @@ namespace EduSpider.Spiders.ClassInRule
{
courseId = courseObj.GetInt("courseId"),
courseName=courseObj.GetString("courseName"),
beginTime=courseObj.GetInt("beginTime"),
});
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment