From e766f27d9d155e6fe09152985e2214dc52c2ba8f Mon Sep 17 00:00:00 2001 From: Ethan Knowlton Date: Fri, 22 Sep 2023 14:22:48 -0400 Subject: [PATCH] seperate parser from code, verify this is working -- then to add ipstack --- cmd/echoip/main.go | 6 +++-- go.mod | 1 + go.sum | 2 ++ http/cache.go | 12 +++++---- http/cache_test.go | 12 +++++---- http/http.go | 67 ++++++++-------------------------------------- http/http_test.go | 37 +++++++++++++++++++++++-- iputil/geo/geo.go | 42 +++++++++++++++++++++++++---- paser/parser.go | 33 +++++++++++++++++++++++ 9 files changed, 137 insertions(+), 75 deletions(-) create mode 100644 paser/parser.go diff --git a/cmd/echoip/main.go b/cmd/echoip/main.go index f3d4c94..9c2cd1d 100644 --- a/cmd/echoip/main.go +++ b/cmd/echoip/main.go @@ -47,12 +47,14 @@ func main() { return } + // open GeoIP files r, err := geo.Open(*countryFile, *cityFile, *asnFile) if err != nil { - log.Fatal(err) + log.Fatal("error") } + cache := http.NewCache(*cacheSize) - server := http.New(r, cache, *profile) + server := http.New(&r, cache, *profile) server.IPHeaders = headers if _, err := os.Stat(*template); err == nil { server.Template = *template diff --git a/go.mod b/go.mod index 040fac8..55d4af8 100644 --- a/go.mod +++ b/go.mod @@ -4,5 +4,6 @@ go 1.13 require ( github.com/oschwald/geoip2-golang v1.5.0 + github.com/qioalice/ipstack v1.0.1 // indirect golang.org/x/sys v0.0.0-20210223212115-eede4237b368 // indirect ) diff --git a/go.sum b/go.sum index f00e6ee..e79f682 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/oschwald/maxminddb-golang v1.8.0 h1:Uh/DSnGoxsyp/KYbY1AuP0tYEwfs0sCph github.com/oschwald/maxminddb-golang v1.8.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/qioalice/ipstack v1.0.1 h1:Ync2O+tR2AH9/TzTg4aeJxd9c1Xz2CeH1joECwnhvms= +github.com/qioalice/ipstack v1.0.1/go.mod h1:6eB9LdNCUdUoOsfDB8Pn2GpmD2I+f2k3yR30ceuf/rY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= diff --git a/http/cache.go b/http/cache.go index 5568d7a..e77cf4c 100644 --- a/http/cache.go +++ b/http/cache.go @@ -6,6 +6,8 @@ import ( "hash/fnv" "net" "sync" + + parser "github.com/mpolden/echoip/paser" ) type Cache struct { @@ -39,7 +41,7 @@ func key(ip net.IP) uint64 { return h.Sum64() } -func (c *Cache) Set(ip net.IP, resp Response) { +func (c *Cache) Set(ip net.IP, resp parser.Response) { if c.capacity == 0 { return } @@ -50,7 +52,7 @@ func (c *Cache) Set(ip net.IP, resp Response) { if minEvictions > 0 { // At or above capacity. Shrink the cache evicted := 0 for el := c.values.Front(); el != nil && evicted < minEvictions; { - value := el.Value.(Response) + value := el.Value.(parser.Response) delete(c.entries, key(value.IP)) next := el.Next() c.values.Remove(el) @@ -66,15 +68,15 @@ func (c *Cache) Set(ip net.IP, resp Response) { c.entries[k] = c.values.PushBack(resp) } -func (c *Cache) Get(ip net.IP) (Response, bool) { +func (c *Cache) Get(ip net.IP) (parser.Response, bool) { k := key(ip) c.mu.RLock() defer c.mu.RUnlock() r, ok := c.entries[k] if !ok { - return Response{}, false + return parser.Response{}, false } - return r.Value.(Response), true + return r.Value.(parser.Response), true } func (c *Cache) Resize(capacity int) error { diff --git a/http/cache_test.go b/http/cache_test.go index e99d480..041adfd 100644 --- a/http/cache_test.go +++ b/http/cache_test.go @@ -4,6 +4,8 @@ import ( "fmt" "net" "testing" + + parser "github.com/mpolden/echoip/paser" ) func TestCacheCapacity(t *testing.T) { @@ -19,10 +21,10 @@ func TestCacheCapacity(t *testing.T) { } for i, tt := range tests { c := NewCache(tt.capacity) - var responses []Response + var responses []parser.Response for i := 0; i < tt.addCount; i++ { ip := net.ParseIP(fmt.Sprintf("192.0.2.%d", i)) - r := Response{IP: ip} + r := parser.Response{IP: ip} responses = append(responses, r) c.Set(ip, r) } @@ -48,7 +50,7 @@ func TestCacheCapacity(t *testing.T) { func TestCacheDuplicate(t *testing.T) { c := NewCache(10) ip := net.ParseIP("192.0.2.1") - response := Response{IP: ip} + response := parser.Response{IP: ip} c.Set(ip, response) c.Set(ip, response) want := 1 @@ -64,7 +66,7 @@ func TestCacheResize(t *testing.T) { c := NewCache(10) for i := 1; i <= 20; i++ { ip := net.ParseIP(fmt.Sprintf("192.0.2.%d", i)) - r := Response{IP: ip} + r := parser.Response{IP: ip} c.Set(ip, r) } if got, want := len(c.entries), 10; got != want { @@ -79,7 +81,7 @@ func TestCacheResize(t *testing.T) { if got, want := c.evictions, uint64(0); got != want { t.Errorf("want %d evictions, got %d", want, got) } - r := Response{IP: net.ParseIP("192.0.2.42")} + r := parser.Response{IP: net.ParseIP("192.0.2.42")} c.Set(r.IP, r) if got, want := len(c.entries), 5; got != want { t.Errorf("want %d entries, got %d", want, got) diff --git a/http/http.go b/http/http.go index d0a1838..d41e7b7 100644 --- a/http/http.go +++ b/http/http.go @@ -11,11 +11,9 @@ import ( "net/http/pprof" - "github.com/mpolden/echoip/iputil" - "github.com/mpolden/echoip/iputil/geo" + parser "github.com/mpolden/echoip/paser" "github.com/mpolden/echoip/useragent" - "math/big" "net" "net/http" "strconv" @@ -32,39 +30,19 @@ type Server struct { LookupAddr func(net.IP) (string, error) LookupPort func(net.IP, uint64) error cache *Cache - gr geo.Reader + parser parser.Parser profile bool Sponsor bool } -type Response struct { - IP net.IP `json:"ip"` - IPDecimal *big.Int `json:"ip_decimal"` - Country string `json:"country,omitempty"` - CountryISO string `json:"country_iso,omitempty"` - CountryEU *bool `json:"country_eu,omitempty"` - RegionName string `json:"region_name,omitempty"` - RegionCode string `json:"region_code,omitempty"` - MetroCode uint `json:"metro_code,omitempty"` - PostalCode string `json:"zip_code,omitempty"` - City string `json:"city,omitempty"` - Latitude float64 `json:"latitude,omitempty"` - Longitude float64 `json:"longitude,omitempty"` - Timezone string `json:"time_zone,omitempty"` - ASN string `json:"asn,omitempty"` - ASNOrg string `json:"asn_org,omitempty"` - Hostname string `json:"hostname,omitempty"` - UserAgent *useragent.UserAgent `json:"user_agent,omitempty"` -} - type PortResponse struct { IP net.IP `json:"ip"` Port uint64 `json:"port"` Reachable bool `json:"reachable"` } -func New(db geo.Reader, cache *Cache, profile bool) *Server { - return &Server{cache: cache, gr: db, profile: profile} +func New(parser parser.Parser, cache *Cache, profile bool) *Server { + return &Server{cache: cache, parser: parser, profile: profile} } func ipFromForwardedForHeader(v string) string { @@ -122,10 +100,10 @@ func userAgentFromRequest(r *http.Request) *useragent.UserAgent { return userAgent } -func (s *Server) newResponse(r *http.Request) (Response, error) { +func (s *Server) newResponse(r *http.Request) (parser.Response, error) { ip, err := ipFromRequest(s.IPHeaders, r, true) if err != nil { - return Response{}, err + return parser.Response{}, err } response, ok := s.cache.Get(ip) if ok { @@ -133,36 +111,12 @@ func (s *Server) newResponse(r *http.Request) (Response, error) { response.UserAgent = userAgentFromRequest(r) return response, nil } - ipDecimal := iputil.ToDecimal(ip) - country, _ := s.gr.Country(ip) - city, _ := s.gr.City(ip) - asn, _ := s.gr.ASN(ip) var hostname string if s.LookupAddr != nil { hostname, _ = s.LookupAddr(ip) } - var autonomousSystemNumber string - if asn.AutonomousSystemNumber > 0 { - autonomousSystemNumber = fmt.Sprintf("AS%d", asn.AutonomousSystemNumber) - } - response = Response{ - IP: ip, - IPDecimal: ipDecimal, - Country: country.Name, - CountryISO: country.ISO, - CountryEU: country.IsEU, - RegionName: city.RegionName, - RegionCode: city.RegionCode, - MetroCode: city.MetroCode, - PostalCode: city.PostalCode, - City: city.Name, - Latitude: city.Latitude, - Longitude: city.Longitude, - Timezone: city.Timezone, - ASN: autonomousSystemNumber, - ASNOrg: asn.AutonomousSystemOrganization, - Hostname: hostname, - } + + response, err = s.parser.Parse(ip, hostname) s.cache.Set(ip, response) response.UserAgent = userAgentFromRequest(r) return response, nil @@ -342,7 +296,7 @@ func (s *Server) DefaultHandler(w http.ResponseWriter, r *http.Request) *appErro } var data = struct { - Response + Response parser.Response Host string BoxLatTop float64 BoxLatBottom float64 @@ -434,7 +388,8 @@ func (s *Server) Handler() http.Handler { r.Route("GET", "/", s.CLIHandler).MatcherFunc(cliMatcher) r.Route("GET", "/", s.CLIHandler).Header("Accept", textMediaType) r.Route("GET", "/ip", s.CLIHandler) - if !s.gr.IsEmpty() { + + if !s.parser.IsEmpty() { r.Route("GET", "/country", s.CLICountryHandler) r.Route("GET", "/country-iso", s.CLICountryISOHandler) r.Route("GET", "/city", s.CLICityHandler) diff --git a/http/http_test.go b/http/http_test.go index 69c9e37..a3adad5 100644 --- a/http/http_test.go +++ b/http/http_test.go @@ -1,6 +1,7 @@ package http import ( + "fmt" "io/ioutil" "log" "net" @@ -10,7 +11,9 @@ import ( "strings" "testing" + "github.com/mpolden/echoip/iputil" "github.com/mpolden/echoip/iputil/geo" + parser "github.com/mpolden/echoip/paser" ) func lookupAddr(net.IP) (string, error) { return "localhost", nil } @@ -32,8 +35,37 @@ func (t *testDb) ASN(net.IP) (geo.ASN, error) { func (t *testDb) IsEmpty() bool { return false } +func (t *testDb) Parse(ip net.IP, hostname string) (parser.Response, error) { + ipDecimal := iputil.ToDecimal(ip) + country, _ := t.Country(ip) + city, _ := t.City(ip) + asn, _ := t.ASN(ip) + var autonomousSystemNumber string + if asn.AutonomousSystemNumber > 0 { + autonomousSystemNumber = fmt.Sprintf("AS%d", asn.AutonomousSystemNumber) + } + return parser.Response{ + IP: ip, + IPDecimal: ipDecimal, + Country: country.Name, + CountryISO: country.ISO, + CountryEU: country.IsEU, + RegionName: city.RegionName, + RegionCode: city.RegionCode, + MetroCode: city.MetroCode, + PostalCode: city.PostalCode, + City: city.Name, + Latitude: city.Latitude, + Longitude: city.Longitude, + Timezone: city.Timezone, + ASN: autonomousSystemNumber, + ASNOrg: asn.AutonomousSystemOrganization, + Hostname: hostname, + }, nil +} + func testServer() *Server { - return &Server{cache: NewCache(100), gr: &testDb{}, LookupAddr: lookupAddr, LookupPort: lookupPort} + return &Server{cache: NewCache(100), parser: &testDb{}, LookupAddr: lookupAddr, LookupPort: lookupPort} } func httpGet(url string, acceptMediaType string, userAgent string) (string, int, error) { @@ -116,7 +148,8 @@ func TestDisabledHandlers(t *testing.T) { server := testServer() server.LookupPort = nil server.LookupAddr = nil - server.gr, _ = geo.Open("", "", "") + parser, _ := geo.Open("", "", "") + server.parser = &parser s := httptest.NewServer(server.Handler()) var tests = []struct { diff --git a/iputil/geo/geo.go b/iputil/geo/geo.go index 4b73729..ae0791d 100644 --- a/iputil/geo/geo.go +++ b/iputil/geo/geo.go @@ -1,9 +1,12 @@ package geo import ( + "fmt" "math" "net" + "github.com/mpolden/echoip/iputil" + parser "github.com/mpolden/echoip/paser" geoip2 "github.com/oschwald/geoip2-golang" ) @@ -42,30 +45,59 @@ type geoip struct { asn *geoip2.Reader } -func Open(countryDB, cityDB string, asnDB string) (Reader, error) { +func Open(countryDB, cityDB string, asnDB string) (geoip, error) { var country, city, asn *geoip2.Reader if countryDB != "" { r, err := geoip2.Open(countryDB) if err != nil { - return nil, err + return geoip{}, err } country = r } if cityDB != "" { r, err := geoip2.Open(cityDB) if err != nil { - return nil, err + return geoip{}, err } city = r } if asnDB != "" { r, err := geoip2.Open(asnDB) if err != nil { - return nil, err + return geoip{}, err } asn = r } - return &geoip{country: country, city: city, asn: asn}, nil + return geoip{country: country, city: city, asn: asn}, nil +} + +func (g *geoip) Parse(ip net.IP, hostname string) (parser.Response, error) { + ipDecimal := iputil.ToDecimal(ip) + country, _ := g.Country(ip) + city, _ := g.City(ip) + asn, _ := g.ASN(ip) + var autonomousSystemNumber string + if asn.AutonomousSystemNumber > 0 { + autonomousSystemNumber = fmt.Sprintf("AS%d", asn.AutonomousSystemNumber) + } + return parser.Response{ + IP: ip, + IPDecimal: ipDecimal, + Country: country.Name, + CountryISO: country.ISO, + CountryEU: country.IsEU, + RegionName: city.RegionName, + RegionCode: city.RegionCode, + MetroCode: city.MetroCode, + PostalCode: city.PostalCode, + City: city.Name, + Latitude: city.Latitude, + Longitude: city.Longitude, + Timezone: city.Timezone, + ASN: autonomousSystemNumber, + ASNOrg: asn.AutonomousSystemOrganization, + Hostname: hostname, + }, nil } func (g *geoip) Country(ip net.IP) (Country, error) { diff --git a/paser/parser.go b/paser/parser.go new file mode 100644 index 0000000..2543c52 --- /dev/null +++ b/paser/parser.go @@ -0,0 +1,33 @@ +package parser + +import ( + "math/big" + "net" + + "github.com/mpolden/echoip/useragent" +) + +type Parser interface { + Parse(net.IP, string) (Response, error) + IsEmpty() bool +} + +type Response struct { + IP net.IP `json:"ip"` + IPDecimal *big.Int `json:"ip_decimal"` + Country string `json:"country,omitempty"` + CountryISO string `json:"country_iso,omitempty"` + CountryEU *bool `json:"country_eu,omitempty"` + RegionName string `json:"region_name,omitempty"` + RegionCode string `json:"region_code,omitempty"` + MetroCode uint `json:"metro_code,omitempty"` + PostalCode string `json:"zip_code,omitempty"` + City string `json:"city,omitempty"` + Latitude float64 `json:"latitude,omitempty"` + Longitude float64 `json:"longitude,omitempty"` + Timezone string `json:"time_zone,omitempty"` + ASN string `json:"asn,omitempty"` + ASNOrg string `json:"asn_org,omitempty"` + Hostname string `json:"hostname,omitempty"` + UserAgent *useragent.UserAgent `json:"user_agent,omitempty"` +}