Skip to content

Commit

Permalink
docs(README.md) - provide some documentation
Browse files Browse the repository at this point in the history
  • Loading branch information
@PxyUp
PxyUp committed Mar 7, 2023
1 parent e1e8f76 commit 030fecd
Show file tree
Hide file tree
Showing 16 changed files with 661 additions and 77 deletions.
634 changes: 609 additions & 25 deletions README.md
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions examples/cli/config_browser.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
"response_type": "xpath",
"connector_type": "browser",
"attempts": 2,
"url": "https://www.theguardian.com/world",
"browser_config": {
"url": "https://www.theguardian.com/world",
"chromium": {
"path": "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
"wait": 10000
Expand Down Expand Up @@ -57,8 +57,8 @@
"response_type": "HTML",
"connector_type": "browser",
"attempts": 3,
"url": "http://www.quotationspage.com/random.php",
"browser_config": {
"url": "http://www.quotationspage.com/random.php",
"chromium": {
"path": "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
"wait": 10000
Expand Down
20 changes: 10 additions & 10 deletions examples/cli/config_cli.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
"connector_config": {
"response_type": "json",
"connector_type": "server",
"url": "https://hacker-news.firebaseio.com/v0/beststories.json?print=pretty&limitToFirst=10&orderBy=%22$key%22",
"server_config": {
"method": "GET",
"url": "https://hacker-news.firebaseio.com/v0/beststories.json?print=pretty&limitToFirst=10&orderBy=%22$key%22"
"method": "GET"
}
},
"model": {
Expand Down Expand Up @@ -44,9 +44,9 @@
"connector_config": {
"response_type": "xpath",
"connector_type": "server",
"url": "https://www.theguardian.com/world",
"server_config": {
"method": "GET",
"url": "https://www.theguardian.com/world"
"method": "GET"
}
}
}
Expand All @@ -72,9 +72,9 @@
"connector_config": {
"response_type": "HTML",
"connector_type": "server",
"url": "http://www.quotationspage.com/random.php",
"server_config": {
"method": "GET",
"url": "http://www.quotationspage.com/random.php"
"method": "GET"
}
}
}
Expand Down Expand Up @@ -171,9 +171,9 @@
"connector_config": {
"response_type": "json",
"connector_type": "server",
"url": "https://hacker-news.firebaseio.com/v0/item/{PL}.json?print=pretty",
"server_config": {
"method": "GET",
"url": "https://hacker-news.firebaseio.com/v0/item/{PL}.json?print=pretty"
"method": "GET"
}
}
}
Expand All @@ -190,9 +190,9 @@
"connector_config": {
"response_type": "json",
"connector_type": "server",
"url": "https://hacker-news.firebaseio.com/v0/item/{PL}.json?print=pretty",
"server_config": {
"method": "GET",
"url": "https://hacker-news.firebaseio.com/v0/item/{PL}.json?print=pretty"
"method": "GET"
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions examples/cli/config_docker.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
"response_type": "xpath",
"connector_type": "browser",
"attempts": 2,
"url": "https://www.theguardian.com/world",
"browser_config": {
"url": "https://www.theguardian.com/world",
"docker": {
"wait": 10000,
"image": "docker.io/zenika/alpine-chrome:with-node",
Expand Down Expand Up @@ -59,8 +59,8 @@
"response_type": "HTML",
"connector_type": "browser",
"attempts": 3,
"url": "http://www.quotationspage.com/random.php",
"browser_config": {
"url": "http://www.quotationspage.com/random.php",
"docker": {
"image": "docker.io/zenika/alpine-chrome",
"wait": 10000,
Expand Down
4 changes: 2 additions & 2 deletions examples/cli/config_playwright.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
"response_type": "xpath",
"connector_type": "browser",
"attempts": 2,
"url": "https://www.theguardian.com/world",
"browser_config": {
"url": "https://www.theguardian.com/world",
"playwright": {
"timeout": 60000,
"wait": 10000,
Expand Down Expand Up @@ -59,8 +59,8 @@
"response_type": "HTML",
"connector_type": "browser",
"attempts": 3,
"url": "http://www.quotationspage.com/random.php",
"browser_config": {
"url": "http://www.quotationspage.com/random.php",
"playwright": {
"timeout": 60000,
"wait": 10000,
Expand Down
8 changes: 4 additions & 4 deletions examples/cli/config_weather.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@
"connector_config": {
"response_type": "HTML",
"connector_type": "server",
"url": "http://www.citymayors.com/gratis/uk_topcities.html",
"server_config": {
"method": "GET",
"url": "http://www.citymayors.com/gratis/uk_topcities.html"
"method": "GET"
}
},
"model": {
Expand Down Expand Up @@ -65,8 +65,8 @@
"response_type": "HTML",
"connector_type": "browser",
"attempts": 4,
"url": "https://openweathermap.org{PL}",
"browser_config": {
"url": "https://openweathermap.org{PL}",
"playwright": {
"timeout": 30,
"wait": 30,
Expand All @@ -87,8 +87,8 @@
"response_type": "xpath",
"connector_type": "browser",
"attempts": 3,
"url": "https://openweathermap.org/find?q={PL}",
"browser_config": {
"url": "https://openweathermap.org/find?q={PL}",
"playwright": {
"timeout": 30,
"wait": 30,
Expand Down
4 changes: 2 additions & 2 deletions examples/config_api.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
"connector_config": {
"response_type": "json",
"connector_type": "server",
"url": "https://chroniclingamerica.loc.gov/search/titles/results/?terms=michigan&format=json",
"server_config": {
"method": "GET",
"url": "https://chroniclingamerica.loc.gov/search/titles/results/?terms=michigan&format=json"
"method": "GET"
}
},
"trigger_config": {
Expand Down
4 changes: 2 additions & 2 deletions examples/config_web.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
"connector_config": {
"response_type": "HTML",
"connector_type": "server",
"url": "https://www.w3schools.com/html/html_examples.asp",
"server_config": {
"method": "GET",
"url": "https://www.w3schools.com/html/html_examples.asp"
"method": "GET"
}
},
"trigger_config": {
Expand Down
2 changes: 1 addition & 1 deletion examples/go/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@ func main() {
ConnectorConfig: &config.ConnectorConfig{
ConnectorType: config.Server,
ResponseType: config.Json,
Url: "https://random-data-api.com/api/appliance/random_appliance",
ServerConfig: &config.ServerConnectorConfig{
Method: http.MethodGet,
Url: "https://random-data-api.com/api/appliance/random_appliance",
},
},
Model: &config.Model{
Expand Down
3 changes: 1 addition & 2 deletions pkg/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ type ConnectorConfig struct {
ResponseType ParserType `json:"response_type" yaml:"response_type"`
ConnectorType Connector `json:"connector_type" yaml:"connector_type"`
ServerConfig *ServerConnectorConfig `json:"server_config" yaml:"server_config"`
Url string `json:"url" yaml:"url"`
BrowserConfig *BrowserConnectorConfig `yaml:"browser_config" json:"browser_config"`
Attempts uint32 `json:"attempts" yaml:"attempts"`
}
Expand All @@ -87,7 +88,6 @@ type PlaywrightConfig struct {
}

type BrowserConnectorConfig struct {
Url string `json:"url" yaml:"url"`
Chromium *ChromiumConfig `json:"chromium" yaml:"chromium"`
Docker *DockerConfig `json:"docker" yaml:"docker"`
Playwright *PlaywrightConfig `json:"playwright" yaml:"playwright"`
Expand All @@ -114,7 +114,6 @@ type ChromiumConfig struct {
type ServerConnectorConfig struct {
Method string `json:"method" yaml:"method"`
Headers map[string]string `yaml:"headers" json:"headers"`
Url string `json:"url" yaml:"url"`
}

type TriggerConfig struct {
Expand Down
12 changes: 7 additions & 5 deletions pkg/connectors/browser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,13 @@ import (
type browserConnector struct {
cfg *config.BrowserConnectorConfig
logger logger.Logger
url string
}

func NewBrowser(cfg *config.BrowserConnectorConfig) *browserConnector {
func NewBrowser(url string, cfg *config.BrowserConnectorConfig) *browserConnector {
return &browserConnector{
cfg: cfg,
url: url,
logger: logger.Null,
}
}
Expand All @@ -23,19 +25,19 @@ func (c *browserConnector) WithLogger(logger logger.Logger) *browserConnector {
}

func (c *browserConnector) Get() ([]byte, error) {
if c.cfg.Url == "" {
if c.url == "" {
return nil, errEmpty
}
if c.cfg.Chromium != nil {
return getFromChromium(c.cfg.Url, c.cfg.Chromium, c.logger.With("emulator", "chromium"))
return getFromChromium(c.url, c.cfg.Chromium, c.logger.With("emulator", "chromium"))
}

if c.cfg.Docker != nil {
return getFromDocker(c.cfg.Url, c.cfg.Docker, c.logger.With("emulator", "docker"))
return getFromDocker(c.url, c.cfg.Docker, c.logger.With("emulator", "docker"))
}

if c.cfg.Playwright != nil {
return getFromPlaywright(c.cfg.Url, c.cfg.Playwright, c.logger.With("emulator", "playwright"))
return getFromPlaywright(c.url, c.cfg.Playwright, c.logger.With("emulator", "playwright"))
}

return nil, nil
Expand Down
4 changes: 2 additions & 2 deletions pkg/connectors/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,11 @@ func init() {
sem = semaphore.NewWeighted(int64(defaultConcurrentRequest))
}

func NewAPI(cfg *config.ServerConnectorConfig, client *http.Client) *apiConnector {
func NewAPI(url string, cfg *config.ServerConnectorConfig, client *http.Client) *apiConnector {
return &apiConnector{
headers: cfg.Headers,
client: client,
url: cfg.Url,
url: url,
method: cfg.Method,
logger: logger.Null,
}
Expand Down
1 change: 1 addition & 0 deletions pkg/parser/json_parser_object_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ func (s *JsonParserObjectSuite) Test_FirstOf() {
}

func (s *JsonParserObjectSuite) Test_StaticArray() {

res, err := s.parser.Parse(&config.Model{
Type: config.ArrayModel,
ArrayConfig: &config.ArrayConfig{
Expand Down
Loading

0 comments on commit 030fecd

Please sign in to comment.