This repository has been archived by the owner on Jul 25, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 1
/
main.go
100 lines (81 loc) · 2.47 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
/*
* This file is part of the dupman/syncer project.
*
* (c) 2022. dupman <info@dupman.cloud>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*
* Written by Temuri Takalandze <me@abgeo.dev>
*/
package main
import (
"fmt"
"log"
"sync"
"github.com/dupman/celery"
"github.com/dupman/encryptor"
"github.com/dupman/sdk/dupman/session"
"github.com/dupman/sdk/service/system"
"github.com/dupman/sdk/service/system/model"
"github.com/dupman/syncer/lib"
"github.com/gocelery/gocelery"
)
func processWebsites(celeryClient *gocelery.CeleryClient, dupmanSession *session.Session) error {
rsaEncryptor := encryptor.NewRSAEncryptor()
err := rsaEncryptor.GenerateKeyPair()
if err != nil {
return fmt.Errorf("error generating key pair: %w", err)
}
publicKey, err := rsaEncryptor.PublicKey()
if err != nil {
return fmt.Errorf("error getting public key: %w", err)
}
systemService := system.New(dupmanSession)
currentPage := 1
totalPages := 1
var wg sync.WaitGroup
for currentPage <= totalPages {
wg.Add(1)
response, err := systemService.GetWebsites(publicKey, currentPage)
if err != nil {
log.Printf("Error getting websites: %s", err)
}
totalPages = response.Pagination.TotalPages
log.Printf("Fetched websites from page %d of %d", currentPage, totalPages)
currentPage++
go func(websites []model.Website, rsaEncryptor *encryptor.RSAEncryptor, celeryClient *gocelery.CeleryClient) {
defer wg.Done()
for _, website := range websites {
token, err := rsaEncryptor.Decrypt(website.Token)
if err != nil {
log.Printf("Failed to decrypt token for website %s", website.URL)
continue
}
_, err = celeryClient.Delay("dupman.website.fetch", website.ID, website.URL, token)
if err != nil {
log.Printf("Failed to create job for website %s", website.URL)
}
}
}(response.Websites, rsaEncryptor, celeryClient)
}
wg.Wait()
return nil
}
func main() {
conf := lib.NewConfig()
// Create new Celery client.
celeryClient, err := celery.NewClient(&conf.Celery)
if err != nil {
log.Fatalf("Failed to create celery client: %s", err)
}
// Create new Dupman client.
dupmanSession, err := session.New(conf.Dupman.Username, conf.Dupman.Password, conf.Dupman.URL)
if err != nil {
log.Fatalf("Failed to create dupman session: %s", err)
}
err = processWebsites(celeryClient, dupmanSession)
if err != nil {
log.Fatalf("Failed to process websites: %s", err)
}
}