wberube
/
echoip
mirror of https://github.com/mpolden/echoip
1
0
Fork 0
Code Issues Releases Wiki Activity

seperate parser from code, verify this is working -- then to add ipstack

This commit is contained in:
Ethan Knowlton 2023-09-22 14:22:48 -04:00
parent d84665c26c
commit e766f27d9d
9 changed files with 137 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/echoip/main.go
View File

@ -47,12 +47,14 @@ func main() {
return
}
// open GeoIP files
r, err := geo.Open(*countryFile, *cityFile, *asnFile)
if err != nil {
log.Fatal(err)
log.Fatal("error")
}
cache := http.NewCache(*cacheSize)
server := http.New(r, cache, *profile)
server := http.New(&r, cache, *profile)
server.IPHeaders = headers
if _, err := os.Stat(*template); err == nil {
server.Template = *template

1
go.mod
View File

@ -4,5 +4,6 @@ go 1.13
require (
github.com/oschwald/geoip2-golang v1.5.0
github.com/qioalice/ipstack v1.0.1 // indirect
golang.org/x/sys v0.0.0-20210223212115-eede4237b368 // indirect
)

2
go.sum
View File

@ -6,6 +6,8 @@ github.com/oschwald/maxminddb-golang v1.8.0 h1:Uh/DSnGoxsyp/KYbY1AuP0tYEwfs0sCph
github.com/oschwald/maxminddb-golang v1.8.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/qioalice/ipstack v1.0.1 h1:Ync2O+tR2AH9/TzTg4aeJxd9c1Xz2CeH1joECwnhvms=
github.com/qioalice/ipstack v1.0.1/go.mod h1:6eB9LdNCUdUoOsfDB8Pn2GpmD2I+f2k3yR30ceuf/rY=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=

12
http/cache.go
View File

@ -6,6 +6,8 @@ import (
"hash/fnv"
"net"
"sync"
parser "github.com/mpolden/echoip/paser"
)
type Cache struct {
@ -39,7 +41,7 @@ func key(ip net.IP) uint64 {
return h.Sum64()
}
func (c *Cache) Set(ip net.IP, resp Response) {
func (c *Cache) Set(ip net.IP, resp parser.Response) {
if c.capacity == 0 {
return
}
@ -50,7 +52,7 @@ func (c *Cache) Set(ip net.IP, resp Response) {
if minEvictions > 0 { // At or above capacity. Shrink the cache
evicted := 0
for el := c.values.Front(); el != nil && evicted < minEvictions; {
value := el.Value.(Response)
value := el.Value.(parser.Response)
delete(c.entries, key(value.IP))
next := el.Next()
c.values.Remove(el)
@ -66,15 +68,15 @@ func (c *Cache) Set(ip net.IP, resp Response) {
c.entries[k] = c.values.PushBack(resp)
}
func (c *Cache) Get(ip net.IP) (Response, bool) {
func (c *Cache) Get(ip net.IP) (parser.Response, bool) {
k := key(ip)
c.mu.RLock()
defer c.mu.RUnlock()
r, ok := c.entries[k]
if !ok {
return Response{}, false
return parser.Response{}, false
}
return r.Value.(Response), true
return r.Value.(parser.Response), true
}
func (c *Cache) Resize(capacity int) error {

12
http/cache_test.go
View File

@ -4,6 +4,8 @@ import (
"fmt"
"net"
"testing"
parser "github.com/mpolden/echoip/paser"
)
func TestCacheCapacity(t *testing.T) {
@ -19,10 +21,10 @@ func TestCacheCapacity(t *testing.T) {
}
for i, tt := range tests {
c := NewCache(tt.capacity)
var responses []Response
var responses []parser.Response
for i := 0; i < tt.addCount; i++ {
ip := net.ParseIP(fmt.Sprintf("192.0.2.%d", i))
r := Response{IP: ip}
r := parser.Response{IP: ip}
responses = append(responses, r)
c.Set(ip, r)
}
@ -48,7 +50,7 @@ func TestCacheCapacity(t *testing.T) {
func TestCacheDuplicate(t *testing.T) {
c := NewCache(10)
ip := net.ParseIP("192.0.2.1")
response := Response{IP: ip}
response := parser.Response{IP: ip}
c.Set(ip, response)
c.Set(ip, response)
want := 1
@ -64,7 +66,7 @@ func TestCacheResize(t *testing.T) {
c := NewCache(10)
for i := 1; i <= 20; i++ {
ip := net.ParseIP(fmt.Sprintf("192.0.2.%d", i))
r := Response{IP: ip}
r := parser.Response{IP: ip}
c.Set(ip, r)
}
if got, want := len(c.entries), 10; got != want {
@ -79,7 +81,7 @@ func TestCacheResize(t *testing.T) {
if got, want := c.evictions, uint64(0); got != want {
t.Errorf("want %d evictions, got %d", want, got)
}
r := Response{IP: net.ParseIP("192.0.2.42")}
r := parser.Response{IP: net.ParseIP("192.0.2.42")}
c.Set(r.IP, r)
if got, want := len(c.entries), 5; got != want {
t.Errorf("want %d entries, got %d", want, got)

67
http/http.go
View File

@ -11,11 +11,9 @@ import (
"net/http/pprof"
"github.com/mpolden/echoip/iputil"
"github.com/mpolden/echoip/iputil/geo"
parser "github.com/mpolden/echoip/paser"
"github.com/mpolden/echoip/useragent"
"math/big"
"net"
"net/http"
"strconv"
@ -32,39 +30,19 @@ type Server struct {
LookupAddr func(net.IP) (string, error)
LookupPort func(net.IP, uint64) error
cache *Cache
gr geo.Reader
parser parser.Parser
profile bool
Sponsor bool
}
type Response struct {
IP net.IP `json:"ip"`
IPDecimal *big.Int `json:"ip_decimal"`
Country string `json:"country,omitempty"`
CountryISO string `json:"country_iso,omitempty"`
CountryEU *bool `json:"country_eu,omitempty"`
RegionName string `json:"region_name,omitempty"`
RegionCode string `json:"region_code,omitempty"`
MetroCode uint `json:"metro_code,omitempty"`
PostalCode string `json:"zip_code,omitempty"`
City string `json:"city,omitempty"`
Latitude float64 `json:"latitude,omitempty"`
Longitude float64 `json:"longitude,omitempty"`
Timezone string `json:"time_zone,omitempty"`
ASN string `json:"asn,omitempty"`
ASNOrg string `json:"asn_org,omitempty"`
Hostname string `json:"hostname,omitempty"`
UserAgent *useragent.UserAgent `json:"user_agent,omitempty"`
}
type PortResponse struct {
IP net.IP `json:"ip"`
Port uint64 `json:"port"`
Reachable bool `json:"reachable"`
}
func New(db geo.Reader, cache *Cache, profile bool) *Server {
return &Server{cache: cache, gr: db, profile: profile}
func New(parser parser.Parser, cache *Cache, profile bool) *Server {
return &Server{cache: cache, parser: parser, profile: profile}
}
func ipFromForwardedForHeader(v string) string {
@ -122,10 +100,10 @@ func userAgentFromRequest(r *http.Request) *useragent.UserAgent {
return userAgent
}
func (s *Server) newResponse(r *http.Request) (Response, error) {
func (s *Server) newResponse(r *http.Request) (parser.Response, error) {
ip, err := ipFromRequest(s.IPHeaders, r, true)
if err != nil {
return Response{}, err
return parser.Response{}, err
}
response, ok := s.cache.Get(ip)
if ok {
@ -133,36 +111,12 @@ func (s *Server) newResponse(r *http.Request) (Response, error) {
response.UserAgent = userAgentFromRequest(r)
return response, nil
}
ipDecimal := iputil.ToDecimal(ip)
country, _ := s.gr.Country(ip)
city, _ := s.gr.City(ip)
asn, _ := s.gr.ASN(ip)
var hostname string
if s.LookupAddr != nil {
hostname, _ = s.LookupAddr(ip)
}
var autonomousSystemNumber string
if asn.AutonomousSystemNumber > 0 {
autonomousSystemNumber = fmt.Sprintf("AS%d", asn.AutonomousSystemNumber)
}
response = Response{
IP: ip,
IPDecimal: ipDecimal,
Country: country.Name,
CountryISO: country.ISO,
CountryEU: country.IsEU,
RegionName: city.RegionName,
RegionCode: city.RegionCode,
MetroCode: city.MetroCode,
PostalCode: city.PostalCode,
City: city.Name,
Latitude: city.Latitude,
Longitude: city.Longitude,
Timezone: city.Timezone,
ASN: autonomousSystemNumber,
ASNOrg: asn.AutonomousSystemOrganization,
Hostname: hostname,
}
response, err = s.parser.Parse(ip, hostname)
s.cache.Set(ip, response)
response.UserAgent = userAgentFromRequest(r)
return response, nil
@ -342,7 +296,7 @@ func (s *Server) DefaultHandler(w http.ResponseWriter, r *http.Request) *appErro
}
var data = struct {
Response
Response parser.Response
Host string
BoxLatTop float64
BoxLatBottom float64
@ -434,7 +388,8 @@ func (s *Server) Handler() http.Handler {
r.Route("GET", "/", s.CLIHandler).MatcherFunc(cliMatcher)
r.Route("GET", "/", s.CLIHandler).Header("Accept", textMediaType)
r.Route("GET", "/ip", s.CLIHandler)
if !s.gr.IsEmpty() {
if !s.parser.IsEmpty() {
r.Route("GET", "/country", s.CLICountryHandler)
r.Route("GET", "/country-iso", s.CLICountryISOHandler)
r.Route("GET", "/city", s.CLICityHandler)

37
http/http_test.go
View File

@ -1,6 +1,7 @@
package http
import (
"fmt"
"io/ioutil"
"log"
"net"
@ -10,7 +11,9 @@ import (
"strings"
"testing"
"github.com/mpolden/echoip/iputil"
"github.com/mpolden/echoip/iputil/geo"
parser "github.com/mpolden/echoip/paser"
)
func lookupAddr(net.IP) (string, error) { return "localhost", nil }
@ -32,8 +35,37 @@ func (t *testDb) ASN(net.IP) (geo.ASN, error) {
func (t *testDb) IsEmpty() bool { return false }
func (t *testDb) Parse(ip net.IP, hostname string) (parser.Response, error) {
ipDecimal := iputil.ToDecimal(ip)
country, _ := t.Country(ip)
city, _ := t.City(ip)
asn, _ := t.ASN(ip)
var autonomousSystemNumber string
if asn.AutonomousSystemNumber > 0 {
autonomousSystemNumber = fmt.Sprintf("AS%d", asn.AutonomousSystemNumber)
}
return parser.Response{
IP: ip,
IPDecimal: ipDecimal,
Country: country.Name,
CountryISO: country.ISO,
CountryEU: country.IsEU,
RegionName: city.RegionName,
RegionCode: city.RegionCode,
MetroCode: city.MetroCode,
PostalCode: city.PostalCode,
City: city.Name,
Latitude: city.Latitude,
Longitude: city.Longitude,
Timezone: city.Timezone,
ASN: autonomousSystemNumber,
ASNOrg: asn.AutonomousSystemOrganization,
Hostname: hostname,
}, nil
}
func testServer() *Server {
return &Server{cache: NewCache(100), gr: &testDb{}, LookupAddr: lookupAddr, LookupPort: lookupPort}
return &Server{cache: NewCache(100), parser: &testDb{}, LookupAddr: lookupAddr, LookupPort: lookupPort}
}
func httpGet(url string, acceptMediaType string, userAgent string) (string, int, error) {
@ -116,7 +148,8 @@ func TestDisabledHandlers(t *testing.T) {
server := testServer()
server.LookupPort = nil
server.LookupAddr = nil
server.gr, _ = geo.Open("", "", "")
parser, _ := geo.Open("", "", "")
server.parser = &parser
s := httptest.NewServer(server.Handler())
var tests = []struct {

42
iputil/geo/geo.go
View File

@ -1,9 +1,12 @@
package geo
import (
"fmt"
"math"
"net"
"github.com/mpolden/echoip/iputil"
parser "github.com/mpolden/echoip/paser"
geoip2 "github.com/oschwald/geoip2-golang"
)
@ -42,30 +45,59 @@ type geoip struct {
asn *geoip2.Reader
}
func Open(countryDB, cityDB string, asnDB string) (Reader, error) {
func Open(countryDB, cityDB string, asnDB string) (geoip, error) {
var country, city, asn *geoip2.Reader
if countryDB != "" {
r, err := geoip2.Open(countryDB)
if err != nil {
return nil, err
return geoip{}, err
}
country = r
}
if cityDB != "" {
r, err := geoip2.Open(cityDB)
if err != nil {
return nil, err
return geoip{}, err
}
city = r
}
if asnDB != "" {
r, err := geoip2.Open(asnDB)
if err != nil {
return nil, err
return geoip{}, err
}
asn = r
}
return &geoip{country: country, city: city, asn: asn}, nil
return geoip{country: country, city: city, asn: asn}, nil
}
func (g *geoip) Parse(ip net.IP, hostname string) (parser.Response, error) {
ipDecimal := iputil.ToDecimal(ip)
country, _ := g.Country(ip)
city, _ := g.City(ip)
asn, _ := g.ASN(ip)
var autonomousSystemNumber string
if asn.AutonomousSystemNumber > 0 {
autonomousSystemNumber = fmt.Sprintf("AS%d", asn.AutonomousSystemNumber)
}
return parser.Response{
IP: ip,
IPDecimal: ipDecimal,
Country: country.Name,
CountryISO: country.ISO,
CountryEU: country.IsEU,
RegionName: city.RegionName,
RegionCode: city.RegionCode,
MetroCode: city.MetroCode,
PostalCode: city.PostalCode,
City: city.Name,
Latitude: city.Latitude,
Longitude: city.Longitude,
Timezone: city.Timezone,
ASN: autonomousSystemNumber,
ASNOrg: asn.AutonomousSystemOrganization,
Hostname: hostname,
}, nil
}
func (g *geoip) Country(ip net.IP) (Country, error) {

33
paser/parser.go Normal file
View File

@ -0,0 +1,33 @@
package parser
import (
"math/big"
"net"
"github.com/mpolden/echoip/useragent"
)
type Parser interface {
Parse(net.IP, string) (Response, error)
IsEmpty() bool
}
type Response struct {
IP net.IP `json:"ip"`
IPDecimal *big.Int `json:"ip_decimal"`
Country string `json:"country,omitempty"`
CountryISO string `json:"country_iso,omitempty"`
CountryEU *bool `json:"country_eu,omitempty"`
RegionName string `json:"region_name,omitempty"`
RegionCode string `json:"region_code,omitempty"`
MetroCode uint `json:"metro_code,omitempty"`
PostalCode string `json:"zip_code,omitempty"`
City string `json:"city,omitempty"`
Latitude float64 `json:"latitude,omitempty"`
Longitude float64 `json:"longitude,omitempty"`
Timezone string `json:"time_zone,omitempty"`
ASN string `json:"asn,omitempty"`
ASNOrg string `json:"asn_org,omitempty"`
Hostname string `json:"hostname,omitempty"`
UserAgent *useragent.UserAgent `json:"user_agent,omitempty"`
}