sandy
/
mop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Parser for the new Yahoo Finance

Browse Source
master
Michael Dvorkin 11 years ago
parent 57fdbbecab
commit 44b1d1fdc9
1 changed files with 133 additions and 91 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 224
      yahoo_market.go

224
yahoo_market.go
Unescape View File

@ -13,7 +13,7 @@ import (
`strings` `strings`
) )
const marketURL = `http://finance.yahoo.com/marketupdate/overview` const marketURL = `http://finance.yahoo.com/` // `http://finance.yahoo.com/marketupdate/overview`
// Market stores current market information displayed in the top three lines of // Market stores current market information displayed in the top three lines of
// the screen. The market data is fetched and parsed from the HTML page above. // the screen. The market data is fetched and parsed from the HTML page above.
@ -22,11 +22,12 @@ type Market struct {
Dow map[string]string // Hash of Dow Jones indicators. Dow map[string]string // Hash of Dow Jones indicators.
Nasdaq map[string]string // Hash of NASDAQ indicators. Nasdaq map[string]string // Hash of NASDAQ indicators.
Sp500 map[string]string // Hash of S&P 500 indicators. Sp500 map[string]string // Hash of S&P 500 indicators.
Advances map[string]string // Number of advanced stocks on NYSE and NASDAQ. London map[string]string
Declines map[string]string // Ditto for declines. Frankfurt map[string]string
Unchanged map[string]string // Ditto for unchanged. Paris map[string]string
Highs map[string]string // Number of new highs on NYSE and NASDAQ. Tokyo map[string]string
Lows map[string]string // Ditto for new lows. HongKong map[string]string
Shanghai map[string]string
regex *regexp.Regexp // Regex to parse market data from HTML. regex *regexp.Regexp // Regex to parse market data from HTML.
errors string // Error(s), if any. errors string // Error(s), if any.
} }
@ -34,33 +35,33 @@ type Market struct {
// Initialize creates empty hashes and builds regular expression used to parse // Initialize creates empty hashes and builds regular expression used to parse
// market data from HTML page. // market data from HTML page.
func (market *Market) Initialize() *Market { func (market *Market) Initialize() *Market {
market.IsClosed = false market.IsClosed = false
market.Dow = make(map[string]string) market.Dow = make(map[string]string)
market.Nasdaq = make(map[string]string) market.Nasdaq = make(map[string]string)
market.Sp500 = make(map[string]string) market.Sp500 = make(map[string]string)
market.Advances = make(map[string]string) market.London = make(map[string]string)
market.Declines = make(map[string]string) market.Frankfurt = make(map[string]string)
market.Unchanged = make(map[string]string) market.Paris = make(map[string]string)
market.Highs = make(map[string]string) market.Tokyo = make(map[string]string)
market.Lows = make(map[string]string) market.HongKong = make(map[string]string)
market.errors = `` market.Shanghai = make(map[string]string)
market.errors = ``
const any = `\s*<.+?>` const any = `\s*<.+?>`
const some = `<.+?` const color = `<.+?price-change-([a-z]+)'>[\+\-]?`
const space = `\s*`
const color = `#([08c]{6});">\s*`
const price = `([\d\.,]+)` const price = `([\d\.,]+)`
const percent = `\(([\d\.,%]+)\)` const percent = `\(([\d\.,%]+)\)`
rules := []string{ rules := []string{
`(Dow)`, any, price, some, color, price, some, percent, any, `S&P 500</span>`, any, price, color, price, any, percent, any,
`(Nasdaq)`, any, price, some, color, price, some, percent, any, `Dow</span>`, any, price, color, price, any, percent, any,
`(S&P 500)`, any, price, some, color, price, some, percent, any, `NASDAQ</span>`, any, price, color, price, any, percent, any,
`(Advances)`, any, price, space, percent, any, price, space, percent, any, `FTSE</span>`, any, price, color, price, any, percent, any,
`(Declines)`, any, price, space, percent, any, price, space, percent, any, `DAX</span>`, any, price, color, price, any, percent, any,
`(Unchanged)`, any, price, space, percent, any, price, space, percent, any, `CAC 40</span>`, any, price, color, price, any, percent, any,
`(New Hi's)`, any, price, any, price, any, `NIKKEI 225</span>`, any, price, color, price, any, percent, any,
`(New Lo's)`, any, price, any, price, any, `Hang Seng</span>`, any, price, color, price, any, percent, any,
`SSE Comp</span>`, any, price, color, price, any, percent, any,
} }
market.regex = regexp.MustCompile(strings.Join(rules, ``)) market.regex = regexp.MustCompile(strings.Join(rules, ``))
@ -101,7 +102,7 @@ func (market *Market) Ok() (bool, string) {
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
func (market *Market) checkIfMarketIsOpen(body []byte) []byte { func (market *Market) checkIfMarketIsOpen(body []byte) []byte {
start := bytes.Index(body, []byte(`id="yfs_market_time"`)) start := bytes.Index(body, []byte(`id='yfs_market_time'`))
finish := start + bytes.Index(body[start:], []byte(`</span>`)) finish := start + bytes.Index(body[start:], []byte(`</span>`))
snippet := body[start:finish] snippet := body[start:finish]
market.IsClosed = bytes.Contains(snippet, []byte(`closed`)) || bytes.Contains(snippet, []byte(`open in`)) market.IsClosed = bytes.Contains(snippet, []byte(`closed`)) || bytes.Contains(snippet, []byte(`open in`))
@ -111,8 +112,8 @@ func (market *Market) checkIfMarketIsOpen(body []byte) []byte {
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
func (market *Market) trim(body []byte) []byte { func (market *Market) trim(body []byte) []byte {
start := bytes.Index(body, []byte(`<table id="yfimktsumm"`)) start := bytes.Index(body, []byte(`>S&P 500<`))
finish := bytes.LastIndex(body, []byte(`<table id="yfimktsumm"`)) finish := bytes.LastIndex(body, []byte(`id="mediafinancesuperherogs"`))
snippet := bytes.Replace(body[start:finish], []byte{'\n'}, []byte{}, -1) snippet := bytes.Replace(body[start:finish], []byte{'\n'}, []byte{}, -1)
snippet = bytes.Replace(snippet, []byte(`&amp;`), []byte{'&'}, -1) snippet = bytes.Replace(snippet, []byte(`&amp;`), []byte{'&'}, -1)
@ -122,77 +123,118 @@ func (market *Market) trim(body []byte) []byte {
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
func (market *Market) extract(snippet []byte) *Market { func (market *Market) extract(snippet []byte) *Market {
matches := market.regex.FindAllStringSubmatch(string(snippet), -1) matches := market.regex.FindAllStringSubmatch(string(snippet), -1)
//fmt.Printf("\n\n\n%q\n\n\n", matches[0])
if len(matches) < 1 || len(matches[0]) < 37 { if len(matches) < 1 || len(matches[0]) < 37 {
panic(`Unable to parse ` + marketURL) panic(`Unable to parse ` + marketURL)
} }
market.Dow[`name`] = matches[0][1] market.Dow[`name`] = `Dow`
market.Dow[`latest`] = matches[0][2] market.Dow[`latest`] = matches[0][1]
market.Dow[`change`] = matches[0][4] market.Dow[`change`] = matches[0][3]
switch matches[0][3] { market.Dow[`percent`] = matches[0][4]
case `008800`: if matches[0][2] == `green` {
market.Dow[`change`] = `+` + matches[0][4] market.Dow[`change`] = `+` + market.Dow[`change`]
market.Dow[`percent`] = `+` + matches[0][5] market.Dow[`percent`] = `+` + market.Dow[`percent`]
case `cc0000`: } else if matches[0][2] == `?` {
market.Dow[`change`] = `-` + matches[0][4] market.Dow[`change`] = `-` + market.Dow[`change`]
market.Dow[`percent`] = `-` + matches[0][5] market.Dow[`percent`] = `-` + market.Dow[`percent`]
default:
market.Dow[`change`] = matches[0][4]
market.Dow[`percent`] = matches[0][5]
} }
market.Nasdaq[`name`] = matches[0][6] market.Nasdaq[`name`] = `NASDAQ`
market.Nasdaq[`latest`] = matches[0][7] market.Nasdaq[`latest`] = matches[0][5]
switch matches[0][8] { market.Nasdaq[`change`] = matches[0][7]
case `008800`: market.Nasdaq[`percent`] = matches[0][8]
market.Nasdaq[`change`] = `+` + matches[0][9] if matches[0][6] == `green` {
market.Nasdaq[`percent`] = `+` + matches[0][10] market.Nasdaq[`change`] = `+` + market.Nasdaq[`change`]
case `cc0000`: market.Nasdaq[`percent`] = `+` + market.Nasdaq[`percent`]
market.Nasdaq[`change`] = `-` + matches[0][9] } else if matches[0][2] == `?` {
market.Nasdaq[`percent`] = `-` + matches[0][10] market.Nasdaq[`change`] = `-` + market.Nasdaq[`change`]
default: market.Nasdaq[`percent`] = `-` + market.Nasdaq[`percent`]
market.Nasdaq[`change`] = matches[0][9]
market.Nasdaq[`percent`] = matches[0][10]
} }
market.Sp500[`name`] = matches[0][11] market.Sp500[`name`] = `S&P 500`
market.Sp500[`latest`] = matches[0][12] market.Sp500[`latest`] = matches[0][9]
switch matches[0][13] { market.Sp500[`change`] = matches[0][11]
case `008800`: market.Sp500[`percent`] = matches[0][12]
market.Sp500[`change`] = `+` + matches[0][14] if matches[0][10] == `green` {
market.Sp500[`percent`] = `+` + matches[0][15] market.Sp500[`change`] = `+` + market.Sp500[`change`]
case `cc0000`: market.Sp500[`percent`] = `+` + market.Sp500[`percent`]
market.Sp500[`change`] = `-` + matches[0][14] } else if matches[0][2] == `?` {
market.Sp500[`percent`] = `-` + matches[0][15] market.Sp500[`change`] = `-` + market.Sp500[`change`]
default: market.Sp500[`percent`] = `-` + market.Sp500[`percent`]
market.Sp500[`change`] = matches[0][14]
market.Sp500[`percent`] = matches[0][15]
} }
market.Advances[`name`] = matches[0][16] market.London[`name`] = `London`
market.Advances[`nyse`] = matches[0][17] market.London[`latest`] = matches[0][13]
market.Advances[`nysep`] = matches[0][18] market.London[`change`] = matches[0][15]
market.Advances[`nasdaq`] = matches[0][19] market.London[`percent`] = matches[0][16]
market.Advances[`nasdaqp`] = matches[0][20] if matches[0][14] == `green` {
market.London[`change`] = `+` + market.London[`change`]
market.Declines[`name`] = matches[0][21] market.London[`percent`] = `+` + market.London[`percent`]
market.Declines[`nyse`] = matches[0][22] } else if matches[0][2] == `?` {
market.Declines[`nysep`] = matches[0][23] market.London[`change`] = `-` + market.London[`change`]
market.Declines[`nasdaq`] = matches[0][24] market.London[`percent`] = `-` + market.London[`percent`]
market.Declines[`nasdaqp`] = matches[0][25] }
market.Unchanged[`name`] = matches[0][26] market.Frankfurt[`name`] = `Frankfurt`
market.Unchanged[`nyse`] = matches[0][27] market.Frankfurt[`latest`] = matches[0][17]
market.Unchanged[`nysep`] = matches[0][28] market.Frankfurt[`change`] = matches[0][19]
market.Unchanged[`nasdaq`] = matches[0][29] market.Frankfurt[`percent`] = matches[0][20]
market.Unchanged[`nasdaqp`] = matches[0][30] if matches[0][18] == `green` {
market.Frankfurt[`change`] = `+` + market.Frankfurt[`change`]
market.Highs[`name`] = matches[0][31] market.Frankfurt[`percent`] = `+` + market.Frankfurt[`percent`]
market.Highs[`nyse`] = matches[0][32] } else if matches[0][2] == `?` {
market.Highs[`nasdaq`] = matches[0][33] market.Frankfurt[`change`] = `-` + market.Frankfurt[`change`]
market.Lows[`name`] = matches[0][34] market.Frankfurt[`percent`] = `-` + market.Frankfurt[`percent`]
market.Lows[`nyse`] = matches[0][35] }
market.Lows[`nasdaq`] = matches[0][36]
market.Paris[`name`] = `Paris`
market.Paris[`latest`] = matches[0][21]
market.Paris[`change`] = matches[0][23]
market.Paris[`percent`] = matches[0][24]
if matches[0][22] == `green` {
market.Paris[`change`] = `+` + market.Paris[`change`]
market.Paris[`percent`] = `+` + market.Paris[`percent`]
} else if matches[0][2] == `?` {
market.Paris[`change`] = `-` + market.Paris[`change`]
market.Paris[`percent`] = `-` + market.Paris[`percent`]
}
market.Tokyo[`name`] = `Tokyo`
market.Tokyo[`latest`] = matches[0][25]
market.Tokyo[`change`] = matches[0][27]
market.Tokyo[`percent`] = matches[0][28]
if matches[0][26] == `green` {
market.Tokyo[`change`] = `+` + market.Tokyo[`change`]
market.Tokyo[`percent`] = `+` + market.Tokyo[`percent`]
} else if matches[0][2] == `?` {
market.Tokyo[`change`] = `-` + market.Tokyo[`change`]
market.Tokyo[`percent`] = `-` + market.Tokyo[`percent`]
}
market.HongKong[`name`] = `Hong Kong`
market.HongKong[`latest`] = matches[0][29]
market.HongKong[`change`] = matches[0][31]
market.HongKong[`percent`] = matches[0][32]
if matches[0][30] == `green` {
market.HongKong[`change`] = `+` + market.HongKong[`change`]
market.HongKong[`percent`] = `+` + market.HongKong[`percent`]
} else if matches[0][2] == `?` {
market.HongKong[`change`] = `-` + market.HongKong[`change`]
market.HongKong[`percent`] = `-` + market.HongKong[`percent`]
}
market.Shanghai[`name`] = `Shanghai`
market.Shanghai[`latest`] = matches[0][33]
market.Shanghai[`change`] = matches[0][36]
market.Shanghai[`percent`] = matches[0][36]
if matches[0][34] == `green` {
market.Shanghai[`change`] = `+` + market.Shanghai[`change`]
market.Shanghai[`percent`] = `+` + market.Shanghai[`percent`]
} else if matches[0][2] == `?` {
market.Shanghai[`change`] = `-` + market.Shanghai[`change`]
market.Shanghai[`percent`] = `-` + market.Shanghai[`percent`]
}
return market return market
} }

Write Preview
Loading…
Cancel
Save