新增反向代理服务器配置,nginx反代替换代理访问yahoo API

nginx.conf

@@ -0,0 +1,37 @@
+# 全局配置
+user www;
+worker_processes auto;
+error_log /www/wwwlogs/nginx_error.log;
+pid /var/run/nginx.pid;
+
+# 事件模块配置
+events {
+    worker_connections 1024;
+}
+
+# HTTP模块配置
+http {
+    include mime.types;
+    default_type application/octet-stream;
+
+    # 日志格式
+    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log off;
+
+    # 代理服务器配置
+    server {
+        listen 80;
+        server_name racknerd.jatus.top;
+
+        location / {
+            proxy_pass https://query1.finance.yahoo.com/;
+            #proxy_set_header Host $host;
+            #proxy_set_header X-Real-IP $remote_addr;
+            #proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_ssl_verify off;
+        }
+    }
+}

yahoo_crumb.go

@@ -9,26 +9,17 @@ import (
 	"net/http"
 	"strings"
 	"net/url"
+	"log"
 )
 
 const crumbURL = "https://query1.finance.yahoo.com/v1/test/getcrumb"
 const cookieURL = "https://login.yahoo.com"
 const userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/113.0"
+const substitute = "http://racknerd.jatus.top"
 
-func fetchCrumb(cookies string) string {
-	proxyUrl, err := url.Parse("http://admin:pavilion@i.jatus.top:9981")
-	if err != nil {
-		panic(err)
-	}
-	
-	transport := &http.Transport{
-		Proxy: http.ProxyURL(proxyUrl),
-	}
-	
-	client := &http.Client{
-		Transport: transport,
-	}
-	//client := http.Client{}
+func fetchCrumb(cookies string) (string, string) {
+	substituteUrl := crumbURL
+	client := http.Client{}
 	request, err := http.NewRequest("GET", crumbURL, nil)
 	if err != nil {
 		panic(err)
@@ -48,7 +39,7 @@ func fetchCrumb(cookies string) string {
 		"TE":              {"trailers"},
 		"User-Agent":      {userAgent},
 	}
-	request.Header.Set("User-Agent", "Android 10; K")
+
 	response, err := client.Do(request)
 	if err != nil {
 		panic(err)
@@ -60,7 +51,29 @@ func fetchCrumb(cookies string) string {
 		panic(err)
 	}
 
-	return string(body[:])
+	if len(body) > 50 {//means error happened
+		// 在请求完成后修改 URL
+		newURL := strings.Replace(substituteUrl, "https://query1.finance.yahoo.com", substitute, -1) // 替换为你想要的新 URL
+		request.URL, err = url.Parse(newURL)
+		if err != nil {
+			panic(err)
+		}
+		log.Println("URL is", request.URL.String())
+		// 发送修改后的请求
+		response, err = client.Do(request)
+		if err != nil {
+			panic(err)
+		}
+		defer response.Body.Close()
+
+		body, err := ioutil.ReadAll(response.Body)
+		if err != nil {
+			panic(err)
+		}
+		log.Println("Crumb:", string(body[:]))
+		return string(body[:]),substitute
+	}
+	return string(body[:]),""
 }
 
 func fetchCookies() string {
@@ -98,5 +111,6 @@ func fetchCookies() string {
 		}
 	}
 	result = strings.TrimSuffix(result, "; ")
+	log.Println("Cookies:", result)
 	return result
 }

yahoo_market.go

@@ -9,8 +9,8 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"net/url"
 	"log"
+	"strings"
 
 	"easyquotation/stock"
 )
@@ -63,9 +63,17 @@ func NewMarket(res map[string]*stock.Stock, watchlist *Watchlist) *Market {
 	market.Gold = make(map[string]string)
 
 	market.cookies = fetchCookies()
-	market.crumb = fetchCrumb(market.cookies)
+	substitute := ""
+	market.crumb, substitute = fetchCrumb(market.cookies)
+	if substitute != "" {
+		newURL := strings.Replace(marketURL, "https://query1.finance.yahoo.com", substitute, -1)
+		log.Println("URL is ", newURL)
+
+		market.url = fmt.Sprintf(newURL, market.crumb, `^DJI,^IXIC,^GSPC,^N225,^HSI,^FTSE,^GDAXI,^TNX,CL=F,CNH=X,EUR=X,GC=F`) + marketURLQueryParts
+		log.Println("full URL is ", market.url)
+	}else{
 		market.url = fmt.Sprintf(marketURL, market.crumb, `^DJI,^IXIC,^GSPC,^N225,^HSI,^FTSE,^GDAXI,^TNX,CL=F,CNH=X,EUR=X,GC=F`) + marketURLQueryParts
-
+	}
 	market.errors = ``
 	market.res = res
 	market.watchlist = watchlist
@@ -84,19 +92,7 @@ func (market *Market) Fetch() (self *Market) {
 		}
 	}()
 	
-	proxyUrl, err := url.Parse("http://admin:pavilion@i.jatus.top:9981")//("http://172 .29.100.107:28888")
-	if err != nil {
-		panic(err)
-	}
-	
-	transport := &http.Transport{
-		Proxy: http.ProxyURL(proxyUrl),
-	}
-	
-	client := &http.Client{
-		Transport: transport,
-	}
-	//client := http.Client{}
+	client := http.Client{}
 	request, err := http.NewRequest("GET", market.url, nil)
 	if err != nil {
 		panic(err)
@@ -118,7 +114,7 @@ func (market *Market) Fetch() (self *Market) {
 		"User-Agent":      {userAgent},
 	}
 
-	request.Header.Set("User-Agent", "Android 10; K")
+	//request.Header.Set("User-Agent", "Android 10; K")
 	response, err := client.Do(request)
 	if err != nil {
 		panic(err)
@@ -207,7 +203,7 @@ func (market *Market) extract(body []byte) *Market {
     	market.Yield[`change`] = q.Time//market.watchlist.Baseon
 	}
 	market.Yield[`latest`] = ""
-	log.Println("here we update the market ,time is", q.Time)
+	//log.Println("here we update the market ,time is", q.Time)
 
     market.Oil = assign(results, 8, true)
     market.Yen = assign(results, 9, true)