refine fetch data

master
sandy 1 year ago
parent 590a97559e
commit bedf84ba06
  1. 50
      yahoo_crumb.go
  2. 5
      yahoo_market.go

@ -8,23 +8,26 @@ import (
"io/ioutil"
"net/http"
"strings"
"net/url"
//"net/url"
"log"
)
const crumbURL = "https://query1.finance.yahoo.com/v1/test/getcrumb"
const cookieURL = "https://login.yahoo.com"
const userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/113.0"
const substitute = "http://racknerd.jatus.top:8000"
const substitute = "https://racknerd.jatus.top"
func fetchCrumb(cookies string) (string, string) {
func fetchCrumb(cookies string, subs string) (string, string) {
substituteUrl := crumbURL
if len(subs) > 0 {
substituteUrl = strings.Replace(substituteUrl, "https://query1.finance.yahoo.com", substitute, -1) // 替换为你想要的新 URL
}
client := http.Client{}
request, err := http.NewRequest("GET", crumbURL, nil)
request, err := http.NewRequest("GET", substituteUrl, nil)
if err != nil {
panic(err)
}
request.Header = http.Header{
"Accept": {"*/*"},
"Accept-Encoding": {"gzip, deflate, br"},
@ -32,50 +35,31 @@ func fetchCrumb(cookies string) (string, string) {
"Connection": {"keep-alive"},
"Content-Type": {"text/plain"},
"Cookie": {cookies},
"Host": {"query1.finance.yahoo.com"},
"Host": {"query1.finance.yahoo.com"},//query1.finance.yahoo.com
"Sec-Fetch-Dest": {"empty"},
"Sec-Fetch-Mode": {"cors"},
"Sec-Fetch-Site": {"same-site"},
"TE": {"trailers"},
"User-Agent": {userAgent},
}
response, err := client.Do(request)
if err != nil {
panic(err)
//panic(err)
return "failed",""
}
defer response.Body.Close()
body, err := ioutil.ReadAll(response.Body)
if err != nil {
panic(err)
//panic(err)
return "failed",""
}
if len(body) > 50 {//means error happened
// 在请求完成后修改 URL
newURL := strings.Replace(substituteUrl, "https://query1.finance.yahoo.com", substitute, -1) // 替换为你想要的新 URL
request.URL, err = url.Parse(newURL)
if err != nil {
panic(err)
}
log.Println("URL is", request.URL.String())
// 发送修改后的请求
response, err = client.Do(request)
if err != nil {
panic(err)
}
defer response.Body.Close()
body, err := ioutil.ReadAll(response.Body)
if err != nil {
panic(err)
}
log.Println("Crumb:", string(body[:]))
//log.Println("body:", string(body[:]))
return string(body[:]),substitute
return "failed",""
}
//log.Println("Originl Crumb:", string(body[:]))
return string(body[:]),""
log.Println("Originl Crumb:", string(body[:]))
return string(body[:]),substitute
}
func fetchCookies() string {

@ -66,7 +66,10 @@ func NewMarket(res map[string]*stock.Stock, watchlist *Watchlist) *Market {
market.cookies = fetchCookies()
//market.quotes = quotes
substitute := ""
market.crumb, substitute = fetchCrumb(market.cookies)
market.crumb, substitute = fetchCrumb(market.cookies, "")
if market.crumb == "failed" {
market.crumb, substitute = fetchCrumb(market.cookies, "subs")
}
if substitute != "" {
newURL := strings.Replace(marketURL, "https://query1.finance.yahoo.com", substitute, -1)
//log.Println("URL is ", newURL)

Loading…
Cancel
Save