Merge branch 'master' of http://gitlab.oytour.com/viitto/spider

src/DotnetSpider.Spiders/Model/RB_Travel_Config.cs

@@ -363,6 +363,11 @@ namespace DotnetSpider.Spiders.Model
 		/// </summary>
 		public string KeyInfo { get; set; }
 
+		/// <summary>
+		/// 直客价
+		/// </summary>
+		public decimal B2CPrice { get; set; }
+
 		/// <summary>
 		/// 行程特色
 		/// </summary>

src/DotnetSpider.Spiders/Program.cs

@@ -3,6 +3,7 @@ using System.Threading.Tasks;
 using DotnetSpider.Spiders.Rules.BestRule;
 using DotnetSpider.Spiders.Rules.BwtRule;
 using DotnetSpider.Spiders.Rules.LionRule;
+using DotnetSpider.Spiders.Rules.SouthSastRule;
 using Serilog;
 using Serilog.Events;
 
@@ -51,7 +52,10 @@ namespace DotnetSpider.Spiders
 			//await new BestService().RunAsync();
 
 			//
-			await new BwtService().RunAsync();
+			//await new BwtService().RunAsync();
+
+			//
+			await new SouthEastServicer().RunAsync();
 
 			Console.WriteLine("End...");
 			Environment.Exit(0);

src/DotnetSpider.Spiders/Rules/LionRule/LionListSpider.cs

@@ -46,7 +46,7 @@ namespace DotnetSpider.Spiders.Rules.LionRule
 		{
 			AddDataFlow(new PageParser());
 			AddDataFlow(new ListTravelParser());
-			var request = new Request("https://travel.liontravel.com/search/grouplist?DepartureID=&ArriveID=--9%2C&GoDateStart=2020-07-16&GoDateEnd=2020-12-31&GroupID=&Keywords=&IsEnsureGroup=false&IsSold=true&ThemeID=&GoStartDays=0&TravelType=1&ScriptTime=&QueryStr=&ArriveText=&TourArea=0&Page=1&SortType=3");
+			var request = new Request("https://travel.liontravel.com/search/grouplist?DepartureID=&ArriveID=--9%2C&GoDateStart=2020-07-30&GoDateEnd=2020-12-31&GroupID=&Keywords=&IsEnsureGroup=false&IsSold=true&ThemeID=&GoStartDays=0&TravelType=1&ScriptTime=&QueryStr=&ArriveText=&TourArea=0&Page=1&SortType=3");
 			await AddRequestsAsync(request);
 		}
 

src/DotnetSpider.Spiders/Rules/LionRule/LionService.cs

@@ -35,8 +35,6 @@ namespace DotnetSpider.Spiders.Rules.LionRule
 
 			//6、持久化数据
 			await LionStorageSpider.RunAsync();
-
-			//Console.WriteLine(JArray.FromObject(TRAVEL_CONFIGS));
 		}
 	}
 }

src/DotnetSpider.Spiders/Rules/SouthSastRule/SouthEastListSpider.cs

@@ -62,6 +62,7 @@ namespace DotnetSpider.Spiders.Rules.SouthSastRule
 		{
 			AddDataFlow(new ListParser());
 			List<Request> requests = new List<Request>();
+			SouthEastRequest.TotalPage = 2;
 			for (int i = 1; i <= SouthEastRequest.TotalPage; i++)
 			{
 				var request = new Request(SouthEastRequest.URL);
@@ -148,14 +149,16 @@ namespace DotnetSpider.Spiders.Rules.SouthSastRule
 					foreach (var item in array)
 					{
 						var x = item.ToObject<JObject>();
-						var exsit = SouthEastService.TRAVEL_CONFIGS.Find(y => y.AgentId == x["prodNo"].ToString());
+						var exsit = SouthEastServicer.TRAVEL_CONFIGS.Find(y => y.AgentId == x["prodNo"].ToString());
 						if (exsit == null || string.IsNullOrEmpty(exsit.AgentId))
 						{
 							var keyInfo = new JObject();
 							keyInfo.Add("code", x["prodNo"].ToString());
 							keyInfo.Add("travelCode", x["travelCode"].ToString());
 							keyInfo.Add("travelName", x["travelName"].ToString());
-							SouthEastService.TRAVEL_CONFIGS.Add(new RB_Travel_Config
+							var price = x["price"].ToObject<JObject>();
+							decimal.TryParse(price["b2cPrice"].ToString(), out decimal b2cPrice);
+							var config = new RB_Travel_Config
 							{
 								RealLink = string.Format("https://trip.settour.com.tw/taiwan/product/{0}", x["prodNo"].ToString()),
 								KeyInfo = keyInfo.ToString(),
@@ -187,8 +190,10 @@ namespace DotnetSpider.Spiders.Rules.SouthSastRule
 								TravelState = 1,
 								UpdateBy = 0,
 								VideoStr = string.Empty,
-								YSeat = 20
-							});
+								YSeat = 20,
+								B2CPrice=b2cPrice
+							};
+							SouthEastServicer.TRAVEL_CONFIGS.Add(config);
 						}
 					}
 				}

src/DotnetSpider.Spiders/Rules/SouthSastRule/SouthEastPageSpider.cs

@@ -121,7 +121,7 @@ namespace DotnetSpider.Spiders.Rules.SouthSastRule
 					var getDtSearchResult = JObject.Parse(rootData["getDtSearchResult"].ToString());
 					var data = JObject.Parse(getDtSearchResult["data"].ToString());
 					Int32.TryParse(data["totalPage"].ToString(), out int totalPage);
-					SouthEastService.RequestConfig.TotalPage = totalPage;
+					SouthEastServicer.RequestConfig.TotalPage = totalPage;
 				}
 				return Task.CompletedTask;
 			}

src/DotnetSpider.Spiders/Rules/SouthSastRule/SouthEastProdHtmlDetailsSpider.cs

@@ -59,7 +59,7 @@ namespace DotnetSpider.Spiders.Rules.SouthSastRule
 		{
 			string url = "https://gql.settour.com.tw/graphql";
 			string origin = "https://tour.settour.com.tw";
-			SouthEastService.TRAVEL_CONFIGS.ForEach(x => {
+			SouthEastServicer.TRAVEL_CONFIGS.ForEach(x => {
 				var keys = JObject.Parse(x.KeyInfo);
 				string _prodNo = keys["code"].ToString();		 
 				var request = new Request(url);
@@ -128,7 +128,7 @@ namespace DotnetSpider.Spiders.Rules.SouthSastRule
 
 					});
 				}
-				foreach (var rootItem in SouthEastService.TRAVEL_CONFIGS)
+				foreach (var rootItem in SouthEastServicer.TRAVEL_CONFIGS)
 				{
 					var keys = JObject.Parse(rootItem.KeyInfo);
 					string prodNo = keys["code"].ToString();
@@ -137,7 +137,7 @@ namespace DotnetSpider.Spiders.Rules.SouthSastRule
 
 					}
 				}
-				SouthEastService.TRAVEL_CONFIGS.ForEach(x =>
+				SouthEastServicer.TRAVEL_CONFIGS.ForEach(x =>
 				{
 					
 					//	x.Title = jObject["prodName"].ToString();

src/DotnetSpider.Spiders/Rules/SouthSastRule/SouthEastService.cs → src/DotnetSpider.Spiders/Rules/SouthSastRule/SouthEastServicer.cs

@@ -9,20 +9,20 @@ namespace DotnetSpider.Spiders.Rules.SouthSastRule
 	/// <summary>
 	/// 东南旅游
 	/// </summary>
-	public class SouthEastService
+	public class SouthEastServicer
 	{
 		public static List<RB_Travel_Config> TRAVEL_CONFIGS = new List<RB_Travel_Config>();
 
 		public static SouthEastRequestConfig RequestConfig = new SouthEastRequestConfig();
 
-		public SouthEastService()
+		public SouthEastServicer()
 		{
 			TRAVEL_CONFIGS = new List<RB_Travel_Config>();
 			RequestConfig = new SouthEastRequestConfig()
 			{
 				URL = "https://gql.settour.com.tw/graphql",
 				StartDate = DateTime.Now.AddDays(1).ToString("yyyyMMdd"),
-				EndDate = DateTime.Now.AddDays(7).ToString("yyyyMMdd"),
+				EndDate = DateTime.Now.AddDays(1).ToString("yyyyMMdd"),
 				TotalPage = 1,
 				Origin= "https://trip.settour.com.tw"
 			};
@@ -46,15 +46,10 @@ namespace DotnetSpider.Spiders.Rules.SouthSastRule
 
 			//await SouthEastProdHtmlDetailsSpider.RunAsync();
 
-			Console.WriteLine("COnfig:" + Newtonsoft.Json.JsonConvert.SerializeObject(TRAVEL_CONFIGS));
+	         //保存数据
+			await SouthEastStorageSpider.RunAsync();
 
 			Console.WriteLine("********* SouthEastService_End  ************");
-
-			////3、补充酒店，餐食等信息
-			//await BwtOtherDetailSpider.RunAsync();
-
-			////4、数据持久化
-			//await BwtStorageSpider.RunAsync();
 		}
 	}
 

src/DotnetSpider.Spiders/Rules/SouthSastRule/SouthEastStorageSpider.cs

+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using DotnetSpider.DataFlow;
+using DotnetSpider.DataFlow.Parser;
+using DotnetSpider.Http;
+using DotnetSpider.Scheduler.Component;
+using DotnetSpider.Spiders.Common;
+using DotnetSpider.Spiders.Model;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Newtonsoft.Json.Linq;
+using Serilog;
+
+namespace DotnetSpider.Spiders.Rules.SouthSastRule
+{
+	/// <summary>
+	/// 东南旅游数据存储
+	/// </summary>
+	public class SouthEastStorageSpider : Spider
+	{
+		public static async Task RunAsync()
+		{
+			var builder = Builder.CreateDefaultBuilder<SouthEastStorageSpider>(options =>
+			{
+				options.UseProxy = false;
+				options.Speed = 1;
+				options.RequestTimeout = 60;
+			});
+			builder.UseSerilog();
+			builder.UseQueueDistinctBfsScheduler<HashSetDuplicateRemover>();
+			await builder.Build().RunAsync();
+		}
+
+		public SouthEastStorageSpider(IOptions<SpiderOptions> options, SpiderServices services, ILogger<Spider> logger) : base(options, services, logger)
+		{
+		}
+
+		protected override async Task InitializeAsync(CancellationToken stoppingToken)
+		{
+			AddDataFlow(new VirtualParser());
+			AddDataFlow(GetDefaultStorage());
+			var request = new Request("https://www.baidu.com");
+			await AddRequestsAsync(request);
+		}
+
+		protected class VirtualParser : DataParser
+		{
+			public VirtualParser()
+			{
+
+			}
+
+			protected override Task Parse(DataContext context)
+			{
+				//不做文档解析，借用context填充自动化持久数据
+				var config = typeof(RB_Travel_Config_Clone);
+				var feature = typeof(RB_Travel_Feature);
+				var dinner = typeof(RB_Journey_Dinner);
+				var hotel = typeof(RB_Journey_Hotel);
+				var scenic = typeof(RB_Journey_Scenic);
+				var title = typeof(RB_Journey_Title);
+				var warmtip = typeof(RB_Journey_Warmtip);
+				var price = typeof(RB_Travel_Price);
+				var other = typeof(RB_TravelPrice_OtherFlight);
+				List<RB_Travel_Config_Clone> travelConfigClones = new List<RB_Travel_Config_Clone>();
+				List<RB_Travel_Feature> travelFeatures = new List<RB_Travel_Feature>();
+				List<RB_Journey_Dinner> journeyDinners = new List<RB_Journey_Dinner>();
+				List<RB_Journey_Hotel> journeyHotels = new List<RB_Journey_Hotel>();
+				List<RB_Journey_Scenic> journeyScenics = new List<RB_Journey_Scenic>();
+				List<RB_Journey_Title> journeyTitles = new List<RB_Journey_Title>();
+				List<RB_Journey_Warmtip> journeyWarmtips = new List<RB_Journey_Warmtip>();
+				List<RB_Travel_Price> prices = new List<RB_Travel_Price>();
+				List<RB_TravelPrice_OtherFlight> otherFlights = new List<RB_TravelPrice_OtherFlight>();
+
+				SouthEastServicer.TRAVEL_CONFIGS.ForEach(x => {
+					if (x.Feature != null)
+					{
+						travelFeatures.Add(x.Feature.Clone());
+					}
+					if (x.JourneyDinners != null && x.JourneyDinners.Count > 0)
+					{
+						journeyDinners.AddRange(x.JourneyDinners.Clone());
+					}
+					if (x.JourneyHotels != null && x.JourneyHotels.Count > 0)
+					{
+						journeyHotels.AddRange(x.JourneyHotels.Clone());
+					}
+					if (x.JourneyScenics != null && x.JourneyScenics.Count > 0)
+					{
+						journeyScenics.AddRange(x.JourneyScenics.Clone());
+					}
+					if (x.JourneyTitles != null && x.JourneyTitles.Count > 0)
+					{
+						journeyTitles.AddRange(x.JourneyTitles.Clone());
+					}
+					if (x.JourneyWarmtips != null && x.JourneyWarmtips.Count > 0)
+					{
+						journeyWarmtips.AddRange(x.JourneyWarmtips.Clone());
+					}
+					if (x.Prices != null && x.Prices.Count > 0)
+					{
+						prices.AddRange(x.Prices.Clone());
+					}
+					if (x.OtherFlights != null && x.OtherFlights.Count > 0)
+					{
+						otherFlights.AddRange(x.OtherFlights.Clone());
+					}
+					var obj = JObject.FromObject(x);
+					travelConfigClones.Add(obj.ToObject<RB_Travel_Config_Clone>());
+				});
+
+				context.AddData(config, travelConfigClones);
+				context.AddData(feature, travelFeatures);
+				context.AddData(dinner, journeyDinners);
+				context.AddData(hotel, journeyHotels);
+				context.AddData(scenic, journeyScenics);
+				context.AddData(title, journeyTitles);
+				context.AddData(warmtip, journeyWarmtips);
+				context.AddData(price, prices);
+				context.AddData(other, otherFlights);
+				return Task.CompletedTask;
+			}
+
+		}
+	}
+}