mirrors/gickup
1
0
Fork 0
mirror of https://github.com/cooperspencer/gickup.git synced 2025-05-11 17:55:34 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

added sublogger to gitea (#178)

Browse source 
* added sublogger to gitea

* implemented the rest of the subloggers
This commit is contained in:
Andreas Wachter 2023-10-04 10:55:24 +02:00 • committed by GitHub
parent f03be6607f
commit fecb4040e6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 204 additions and 346 deletions
Download patch file Download diff file Expand all files Collapse all files

32
bitbucket/bitbucket.go
View file

@ -4,9 +4,14 @@ import (
"net/url"
"time"
"github.com/cooperspencer/gickup/logger"
"github.com/cooperspencer/gickup/types"
"github.com/ktrysmt/go-bitbucket"
"github.com/rs/zerolog/log"
"github.com/rs/zerolog"
)
var (
sub zerolog.Logger
)
// Get TODO.
@ -22,12 +27,12 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
if repo.URL == "" {
repo.URL = bitbucket.DEFAULT_BITBUCKET_API_BASE_URL
sub = logger.CreateSubLogger("stage", "bitbucket", "url", repo.URL)
} else {
bitbucketURL, err := url.Parse(repo.URL)
sub = logger.CreateSubLogger("stage", "bitbucket", "url", repo.URL)
if err != nil {
log.Error().
Str("stage", "bitbucket").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
continue
}
@ -36,22 +41,16 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
err := repo.Filter.ParseDuration()
if err != nil {
log.Error().
Str("stage", "bitbucket").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
}
log.Info().
Str("stage", "bitbucket").
Str("url", repo.URL).
sub.Info().
Msgf("grabbing repositories from %s", repo.User)
repositories, err := client.Repositories.ListForAccount(&bitbucket.RepositoriesOptions{Owner: repo.User})
if err != nil {
log.Error().
Str("stage", "bitbucket").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
continue
}
@ -67,11 +66,8 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
}
}
updated, err := time.Parse(time.RFC3339, r.UpdatedOn)
if err == nil {
if time.Since(updated) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 {
continue
}
if time.Since(*r.UpdatedOnTime) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 {
continue
}
if include[r.Name] {

106
gitea/gitea.go
View file

@ -5,8 +5,13 @@ import (
"time"
"code.gitea.io/sdk/gitea"
"github.com/cooperspencer/gickup/logger"
"github.com/cooperspencer/gickup/types"
"github.com/rs/zerolog/log"
"github.com/rs/zerolog"
)
var (
sub zerolog.Logger
)
func getOrgVisibility(visibility string) gitea.VisibleType {
@ -40,23 +45,19 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
if d.URL == "" {
d.URL = "https://gitea.com/"
}
log.Info().
Str("stage", "gitea").
Str("url", d.URL).
sub = logger.CreateSubLogger("stage", "gitea", "url", d.URL)
sub.Info().
Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL)
giteaclient, err := gitea.NewClient(d.URL, gitea.SetToken(d.GetToken()))
if err != nil {
log.Error().Str("stage", "gitea").Str("url", d.URL).Msg(err.Error())
sub.Error().Msg(err.Error())
return false
}
user, _, err := giteaclient.GetMyUserInfo()
if err != nil {
log.Error().
Str("stage", "gitea").
Str("url", d.URL).
sub.Error().
Msg(err.Error())
return false
}
@ -74,18 +75,14 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
Visibility: orgvisibilty,
})
if err != nil {
log.Error().
Str("stage", "gitea").
Str("url", d.URL).
sub.Error().
Msg(err.Error())
return false
}
user.ID = org.ID
user.UserName = org.UserName
} else {
log.Error().
Str("stage", "gitea").
Str("url", d.URL).
sub.Error().
Msg(err.Error())
return false
}
@ -130,17 +127,13 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
_, _, err := giteaclient.MigrateRepo(opts)
if err != nil {
log.Error().
Str("stage", "gitea").
Str("url", d.URL).
sub.Error().
Msg(err.Error())
log.Info().
Str("stage", "gitea").
Str("url", d.URL).
sub.Info().
Msgf("deleting %s again", types.Blue(r.Name))
_, err = giteaclient.DeleteRepo(user.UserName, r.Name)
if err != nil {
log.Error().
sub.Error().
Str("stage", "gitea").
Str("url", d.URL).
Msgf("couldn't delete %s!", types.Red(r.Name))
@ -148,9 +141,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
return false
}
log.Info().
Str("stage", "gitea").
Str("url", d.URL).
sub.Info().
Msgf("mirrored %s to %s", types.Blue(r.Name), d.URL)
return true
@ -159,30 +150,26 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
if d.MirrorInterval != repo.MirrorInterval {
_, _, err := giteaclient.EditRepo(user.UserName, r.Name, gitea.EditRepoOption{MirrorInterval: &d.MirrorInterval})
if err != nil {
log.Error().
Str("stage", "gitea").
Str("url", d.URL).
sub.Error().
Msgf("Couldn't update %s", types.Red(r.Name))
}
return false
}
if repo.Mirror {
log.Info().
Str("stage", "gitea").
Str("url", d.URL).
sub.Info().
Msgf("mirror of %s already exists, syncing instead", types.Blue(r.Name))
_, err := giteaclient.MirrorSync(user.UserName, repo.Name)
if err != nil {
log.Error().
sub.Error().
Str("stage", "gitea").
Str("url", d.URL).
Msg(err.Error())
return false
}
log.Info().
sub.Info().
Str("stage", "gitea").
Str("url", d.URL).
Msgf("successfully synced %s.", types.Blue(r.Name))
@ -196,26 +183,21 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
ran := false
repos := []types.Repo{}
for _, repo := range conf.Source.Gitea {
err := repo.Filter.ParseDuration()
if err != nil {
log.Error().
Str("stage", "gitea").
Str("url", repo.URL).
Msg(err.Error())
}
ran = true
if repo.URL == "" {
repo.URL = "https://gitea.com"
}
sub = logger.CreateSubLogger("stage", "gitea", "url", repo.URL)
err := repo.Filter.ParseDuration()
if err != nil {
sub.Error().
Msg(err.Error())
}
ran = true
if repo.User == "" {
log.Info().
Str("stage", "gitea").
Str("url", repo.URL).
sub.Info().
Msg("grabbing my repositories")
} else {
log.Info().
Str("stage", "gitea").
Str("url", repo.URL).
sub.Info().
Msgf("grabbing repositories from %s", repo.User)
}
opt := gitea.ListReposOptions{}
@ -234,9 +216,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
if token != "" && repo.User == "" {
user, _, err := client.GetMyUserInfo()
if err != nil {
log.Error().
Str("stage", "gitea").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
continue
}
@ -244,9 +224,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
}
if err != nil {
log.Error().
Str("stage", "gitea").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
continue
}
@ -254,9 +232,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
for {
repos, _, err := client.ListUserRepos(repo.User, opt)
if err != nil {
log.Error().
Str("stage", "gitea").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
continue
}
@ -270,9 +246,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
if repo.Starred {
starredrepos, _, err := client.GetStarredRepos(repo.User)
if err != nil {
log.Error().
Str("stage", "gitea").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
} else {
gitearepos = append(gitearepos, starredrepos...)
@ -303,9 +277,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
if len(repo.Filter.Languages) > 0 {
langs, _, err := client.GetRepoLanguages(r.Owner.UserName, r.Name)
if err != nil {
log.Error().
Str("stage", "gitea").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
continue
} else {
@ -398,9 +370,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
for {
o, _, err := client.ListUserOrgs(repo.User, gitea.ListOrgsOptions{ListOptions: orgopt})
if err != nil {
log.Error().
Str("stage", "gitea").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
}
if len(o) == 0 {
@ -450,9 +420,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
if len(repo.Filter.Languages) > 0 {
langs, _, err := client.GetRepoLanguages(r.Owner.UserName, r.Name)
if err != nil {
log.Error().
Str("stage", "gitea").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
continue
} else {
@ -550,7 +518,7 @@ func getOrgRepos(client *gitea.Client, org *gitea.Organization,
o, _, err := client.ListOrgRepos(org.UserName,
gitea.ListOrgReposOptions{orgopt})
if err != nil {
log.Error().Str("stage", "gitea").Str("url", repo.URL).Msg(err.Error())
sub.Error().Str("stage", "gitea").Str("url", repo.URL).Msg(err.Error())
}
return o

40
github/github.go
View file

@ -5,9 +5,10 @@ import (
"strings"
"time"
"github.com/cooperspencer/gickup/logger"
"github.com/cooperspencer/gickup/types"
"github.com/google/go-github/v41/github"
"github.com/rs/zerolog/log"
"github.com/rs/zerolog"
"github.com/shurcooL/githubv4"
"golang.org/x/oauth2"
)
@ -39,6 +40,10 @@ type V4Repo struct {
Repository string
}
var (
sub zerolog.Logger
)
func getv4(token, user string) []V4Repo {
repos := []V4Repo{}
tokenSource := oauth2.StaticTokenSource(
@ -55,8 +60,7 @@ func getv4(token, user string) []V4Repo {
for {
err := client.Query(context.Background(), &query, variables)
if err != nil {
log.Error().
Str("stage", "github").
sub.Error().
Msg(err.Error())
return []V4Repo{}
}
@ -99,23 +103,18 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
ran := false
repos := []types.Repo{}
for _, repo := range conf.Source.Github {
sub = logger.CreateSubLogger("stage", "github", "url", "https://github.com")
err := repo.Filter.ParseDuration()
if err != nil {
log.Error().
Str("stage", "github").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
}
ran = true
if repo.User == "" {
log.Info().
Str("stage", "github").
Str("url", "https://github.com").
sub.Info().
Msg("grabbing my repositories")
} else {
log.Info().
Str("stage", "github").
Str("url", "https://github.com").
sub.Info().
Msgf("grabbing the repositories from %s", repo.User)
}
@ -145,9 +144,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
if token != "" {
user, _, err := client.Users.Get(context.TODO(), "")
if err != nil {
log.Error().
Str("stage", "github").
Str("url", "https://github.com").
sub.Error().
Msg(err.Error())
continue
}
@ -162,9 +159,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
for _, r := range getv4(token, v4user) {
github_repo, _, err := client.Repositories.Get(context.Background(), r.User, r.Repository)
if err != nil {
log.Error().
Str("stage", "github").
Str("url", "https://github.com").
sub.Error().
Msg(err.Error())
continue
}
@ -176,9 +171,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
opt.Page = i
repos, _, err := client.Repositories.List(context.TODO(), repo.User, opt)
if err != nil {
log.Error().
Str("stage", "github").
Str("url", "https://github.com").
sub.Error().
Msg(err.Error())
continue
}
@ -202,9 +195,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
opt.ListOptions.Page = i
repos, _, err := client.Activity.ListStarred(context.TODO(), repo.User, opt)
if err != nil {
log.Error().
Str("stage", "github").
Str("url", "https://github.com").
sub.Error().
Msg(err.Error())
}
if len(repos) == 0 {
@ -328,6 +319,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
}
func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) {
sub = logger.CreateSubLogger("stage", "github", "url", "https://github.com")
token := destination.GetToken()
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: token},

67
gitlab/gitlab.go
View file

@ -6,11 +6,16 @@ import (
"strings"
"time"
"github.com/cooperspencer/gickup/logger"
"github.com/cooperspencer/gickup/types"
"github.com/rs/zerolog/log"
"github.com/rs/zerolog"
"github.com/xanzy/go-gitlab"
)
var (
sub zerolog.Logger
)
// Backup TODO.
func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
var gitlabclient *gitlab.Client
@ -22,18 +27,15 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
} else {
gitlabclient, err = gitlab.NewClient(token, gitlab.WithBaseURL(d.URL))
}
sub = logger.CreateSubLogger("stage", "gitlab", "url", d.URL)
if err != nil {
log.Error().
Str("stage", "gitlab").
Str("url", d.URL).
sub.Error().
Msg(err.Error())
return false
}
log.Info().
Str("stage", "gitlab").
Str("url", d.URL).
sub.Info().
Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL)
True := true
@ -45,7 +47,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
projects, _, err := gitlabclient.Projects.ListProjects(&opt)
if err != nil {
log.Error().Str("stage", "gitlab").Str("url", d.URL).Msg(err.Error())
sub.Error().Msg(err.Error())
return false
}
@ -85,9 +87,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
_, _, err = gitlabclient.Projects.CreateProject(opts)
if err != nil {
log.Error().
Str("stage", "gitlab").
Str("url", d.URL).
sub.Error().
Msg(err.Error())
return false
}
@ -101,10 +101,9 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
repos := []types.Repo{}
for _, repo := range conf.Source.Gitlab {
err := repo.Filter.ParseDuration()
sub = logger.CreateSubLogger("stage", "gitlab", "url", repo.URL)
if err != nil {
log.Error().
Str("stage", "gitlab").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
}
ran = true
@ -112,18 +111,14 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
repo.URL = "https://gitlab.com"
}
log.Info().
Str("stage", "gitlab").
Str("url", repo.URL).
sub.Info().
Msgf("grabbing repositories from %s", repo.User)
gitlabrepos := []*gitlab.Project{}
gitlabgrouprepos := map[string][]*gitlab.Project{}
token := repo.GetToken()
client, err := gitlab.NewClient(token, gitlab.WithBaseURL(repo.URL))
if err != nil {
log.Error().
Str("stage", "gitlab").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
continue
}
@ -131,9 +126,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
opt := &gitlab.ListProjectsOptions{}
users, _, err := client.Users.ListUsers(&gitlab.ListUsersOptions{Username: &repo.User})
if err != nil {
log.Error().
Str("stage", "gitlab").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
continue
}
@ -146,9 +139,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
opt.Page = i
projects, _, err := client.Projects.ListUserProjects(user.ID, opt)
if err != nil {
log.Error().
Str("stage", "gitlab").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
}
if len(projects) == 0 {
@ -168,9 +159,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
opt.Page = i
projects, _, err := client.Projects.ListUserStarredProjects(user.ID, opt)
if err != nil {
log.Error().
Str("stage", "gitlab").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
}
if len(projects) == 0 {
@ -204,9 +193,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
if len(repo.Filter.Languages) > 0 {
langs, _, err := client.Projects.GetProjectLanguages(r.ID)
if err != nil {
log.Error().
Str("stage", "gitlab").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
continue
} else {
@ -318,9 +305,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
},
})
if err != nil {
log.Error().
Str("stage", "gitlab").
Str("url", repo.URL).Msg(err.Error())
sub.Error().Msg(err.Error())
}
if len(g) == 0 {
@ -339,9 +324,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
for {
projects, _, err := client.Groups.ListGroupProjects(group.ID, gopt)
if err != nil {
log.Error().
Str("stage", "gitlab").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
}
if len(projects) == 0 {
@ -374,9 +357,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
if len(repo.Filter.Languages) > 0 {
langs, _, err := client.Projects.GetProjectLanguages(r.ID)
if err != nil {
log.Error().
Str("stage", "gitlab").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
continue
} else {
@ -496,9 +477,7 @@ func activeWiki(r *gitlab.Project, client *gitlab.Client, repo types.GenRepo) bo
wikis, _, err := client.Wikis.ListWikis(r.ID, wikilistoptions)
if err != nil {
log.Warn().
Str("stage", "gitlab").
Str("url", repo.URL).
sub.Warn().
Msg(err.Error())
}

77
gogs/gogs.go
View file

@ -3,9 +3,14 @@ package gogs
import (
"time"
"github.com/cooperspencer/gickup/logger"
"github.com/cooperspencer/gickup/types"
"github.com/gogs/go-gogs-client"
"github.com/rs/zerolog/log"
"github.com/rs/zerolog"
)
var (
sub zerolog.Logger
)
func getRepoVisibility(visibility string, private bool) bool {
@ -22,18 +27,15 @@ func getRepoVisibility(visibility string, private bool) bool {
// Backup TODO.
func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
repovisibility := getRepoVisibility(d.Visibility.Repositories, r.Private)
log.Info().
Str("stage", "gogs").
Str("url", d.URL).
sub = logger.CreateSubLogger("stage", "gogs", "url", d.URL)
sub.Info().
Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL)
gogsclient := gogs.NewClient(d.URL, d.GetToken())
user, err := gogsclient.GetSelfInfo()
if err != nil {
log.Error().
Str("stage", "gogs").
Str("url", d.URL).
sub.Error().
Msg(err.Error())
return false
}
@ -50,18 +52,14 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
UserName: d.User,
})
if err != nil {
log.Error().
Str("stage", "gogs").
Str("url", d.URL).
sub.Error().
Msg(err.Error())
return false
}
user.ID = org.ID
user.UserName = org.UserName
} else {
log.Error().
Str("stage", "gogs").
Str("url", d.URL).
sub.Error().
Msg(err.Error())
return false
}
@ -99,19 +97,13 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
_, err := gogsclient.MigrateRepo(opts)
if err != nil {
log.Error().
Str("stage", "gogs").
Str("url", d.URL).
sub.Error().
Msg(err.Error())
log.Info().
Str("stage", "gogs").
Str("url", d.URL).
sub.Info().
Msgf("deleting %s again", types.Blue(r.Name))
err = gogsclient.DeleteRepo(user.UserName, r.Name)
if err != nil {
log.Error().
Str("stage", "gogs").
Str("url", d.URL).
sub.Error().
Msgf("couldn't delete %s!", types.Red(r.Name))
}
return false
@ -121,23 +113,17 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
}
if repo.Mirror {
log.Info().
Str("stage", "gogs").
Str("url", d.URL).
sub.Info().
Msgf("mirror of %s already exists, syncing instead", types.Blue(r.Name))
err := gogsclient.MirrorSync(user.UserName, repo.Name)
if err != nil {
log.Error().
Str("stage", "gogs").
Str("url", d.URL).
sub.Error().
Msg(err.Error())
return false
}
log.Info().
Str("stage", "gogs").
Str("url", d.URL).
sub.Info().
Msgf("successfully synced %s.", types.Blue(r.Name))
}
@ -149,23 +135,18 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
ran := false
repos := []types.Repo{}
for _, repo := range conf.Source.Gogs {
sub = logger.CreateSubLogger("stage", "gogs", "url", repo.URL)
err := repo.Filter.ParseDuration()
if err != nil {
log.Error().
Str("stage", "gogs").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
}
ran = true
if repo.User == "" {
log.Info().
Str("stage", "gogs").
Str("url", repo.URL).
sub.Info().
Msg("grabbing my repositories")
} else {
log.Info().
Str("stage", "gogs").
Str("url", repo.URL).
sub.Info().
Msgf("grabbing repositories from %s", repo.User)
}
@ -179,9 +160,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
gogsrepos, err = client.ListUserRepos(repo.User)
}
if err != nil {
log.Error().
Str("stage", "gogs").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
continue
}
@ -276,9 +255,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
orgs, err = client.ListUserOrgs(repo.User)
}
if err != nil {
log.Error().
Str("stage", "gogs").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
}
@ -292,9 +269,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
if includeorgs[org.UserName] {
o, err := client.ListOrgRepos(org.UserName)
if err != nil {
log.Error().
Str("stage", "gogs").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
}
@ -307,9 +282,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
} else {
o, err := client.ListOrgRepos(org.UserName)
if err != nil {
log.Error().
Str("stage", "gogs").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
}

112
local/local.go
View file

@ -12,6 +12,7 @@ import (
"time"
"github.com/cooperspencer/gickup/gitcmd"
"github.com/cooperspencer/gickup/logger"
"github.com/cooperspencer/gickup/types"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
@ -20,22 +21,22 @@ import (
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/melbahja/goph"
"github.com/mholt/archiver/v3"
"github.com/rs/zerolog/log"
"github.com/rs/zerolog"
gossh "golang.org/x/crypto/ssh"
)
var (
gitc = gitcmd.GitCmd{}
sub zerolog.Logger
)
// Locally TODO.
func Locally(repo types.Repo, l types.Local, dry bool) bool {
sub = logger.CreateSubLogger("stage", "locally", "path", l.Path)
if l.LFS {
g, err := gitcmd.New()
if err != nil {
log.Error().
Str("stage", "locally").
Str("path", l.Path).
sub.Error().
Msg(err.Error())
}
gitc = g
@ -57,9 +58,7 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool {
stat, err := os.Stat(l.Path)
if os.IsNotExist(err) && !dry {
if err := os.MkdirAll(l.Path, 0o777); err != nil {
log.Error().
Str("stage", "locally").
Str("path", l.Path).
sub.Error().
Msg(err.Error())
return false
}
@ -69,9 +68,7 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool {
if stat != nil && stat.IsDir() {
if err := os.Chdir(l.Path); err != nil {
log.Error().
Str("stage", "locally").
Str("path", l.Path).
sub.Error().
Msg(err.Error())
return false
}
@ -90,9 +87,7 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool {
auth, err = ssh.NewPublicKeysFromFile("git", repo.Origin.SSHKey, "")
if err != nil {
log.Error().
Str("stage", "locally").
Str("path", l.Path).
sub.Error().
Msg(err.Error())
return false
}
@ -111,25 +106,19 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool {
for x := 1; x <= tries; x++ {
stat, err := os.Stat(repo.Name)
if os.IsNotExist(err) {
log.Info().
Str("stage", "locally").
Str("path", l.Path).
sub.Info().
Msgf("cloning %s", types.Green(repo.Name))
err := cloneRepository(repo, auth, dry, l)
if err != nil {
if err.Error() == "repository not found" {
log.Warn().
Str("stage", "locally").
Str("path", l.Path).
sub.Warn().
Str("repo", repo.Name).
Msg(err.Error())
break
}
if x == tries {
log.Warn().
Str("stage", "locally").
Str("path", l.Path).
sub.Warn().
Str("repo", repo.Name).
Msg(err.Error())
@ -137,9 +126,7 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool {
}
if strings.Contains(err.Error(), "ERR access denied or repository not exported") {
log.Warn().
Str("stage", "locally").
Str("path", l.Path).
sub.Warn().
Str("repo", repo.Name).
Msgf("%s doesn't exist.", repo.Name)
@ -147,18 +134,14 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool {
}
if strings.Contains(err.Error(), "remote repository is empty") {
log.Warn().
Str("stage", "locally").
Str("path", l.Path).
sub.Warn().
Str("repo", repo.Name).
Msg(err.Error())
break
}
log.Warn().
Str("stage", "locally").
Str("path", l.Path).
sub.Warn().
Msgf("retry %s from %s", types.Red(x), types.Red(tries))
time.Sleep(5 * time.Second)
@ -167,36 +150,26 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool {
}
} else {
if !stat.IsDir() {
log.Warn().
Str("stage", "locally").
Str("path", l.Path).
sub.Warn().
Str("repo", repo.Name).
Msgf("%s is a file", types.Red(repo.Name))
} else {
log.Info().
Str("stage", "locally").
Str("path", l.Path).
sub.Info().
Msgf("opening %s locally", types.Green(repo.Name))
err := updateRepository(repo.Name, auth, dry, l)
if err != nil {
if err == git.NoErrAlreadyUpToDate {
log.Info().
Str("stage", "locally").
Str("path", l.Path).
sub.Info().
Msg(err.Error())
} else {
if x == tries {
log.Fatal().
Str("stage", "locally").
Str("path", l.Path).
sub.Fatal().
Str("repo", repo.Name).
Msg(err.Error())
} else {
os.RemoveAll(repo.Name)
log.Warn().
Str("stage", "locally").
Str("path", l.Path).
sub.Warn().
Str("repo", repo.Name).
Msgf("retry %s from %s", types.Red(x), types.Red(tries))
@ -210,22 +183,16 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool {
}
if l.Zip {
log.Info().
Str("stage", "locally").
Str("path", l.Path).
sub.Info().
Msgf("zipping %s", types.Green(repo.Name))
err := archiver.Archive([]string{repo.Name}, fmt.Sprintf("%s.zip", repo.Name))
if err != nil {
log.Warn().
Str("stage", "locally").
Str("path", l.Path).
sub.Warn().
Str("repo", repo.Name).Msg(err.Error())
}
err = os.RemoveAll(repo.Name)
if err != nil {
log.Warn().
Str("stage", "locally").
Str("path", l.Path).
sub.Warn().
Str("repo", repo.Name).Msg(err.Error())
}
}
@ -234,9 +201,7 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool {
parentdir := path.Dir(repo.Name)
files, err := os.ReadDir(parentdir)
if err != nil {
log.Warn().
Str("stage", "locally").
Str("path", l.Path).
sub.Warn().
Str("repo", repo.Name).Msg(err.Error())
break
}
@ -249,9 +214,7 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool {
}
_, err := strconv.ParseInt(fname, 10, 64)
if err != nil {
log.Warn().
Str("stage", "locally").
Str("path", l.Path).
sub.Warn().
Str("repo", repo.Name).
Msgf("couldn't parse timestamp! %s", types.Red(file.Name()))
}
@ -266,15 +229,11 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool {
if len(keep) > l.Keep {
toremove := keep[l.Keep:]
for _, file := range toremove {
log.Info().
Str("stage", "locally").
Str("path", l.Path).
sub.Info().
Msgf("removing %s", types.Red(path.Join(parentdir, file)))
err := os.RemoveAll(path.Join(parentdir, file))
if err != nil {
log.Warn().
Str("stage", "locally").
Str("path", l.Path).
sub.Warn().
Str("repo", repo.Name).Msg(err.Error())
}
}
@ -299,8 +258,7 @@ func updateRepository(repoPath string, auth transport.AuthMethod, dry bool, l ty
return err
}
log.Info().
Str("stage", "locally").
sub.Info().
Msgf("pulling %s", types.Green(repoPath))
err = gitc.Pull(l.Bare)
@ -316,8 +274,7 @@ func updateRepository(repoPath string, auth transport.AuthMethod, dry bool, l ty
return err
}
log.Info().
Str("stage", "locally").
sub.Info().
Msgf("pulling %s", types.Green(repoPath))
err = w.Pull(&git.PullOptions{Auth: auth, RemoteName: "origin", SingleBranch: false})
@ -342,17 +299,17 @@ func cloneRepository(repo types.Repo, auth transport.AuthMethod, dry bool, l typ
err := site.GetValues(url)
if err != nil {
log.Fatal().Str("stage", "locally").Str("repo", repo.Name).Msg(err.Error())
sub.Fatal().Str("repo", repo.Name).Msg(err.Error())
}
sshAuth, err := goph.Key(repo.Origin.SSHKey, "")
if err != nil {
log.Fatal().Str("stage", "locally").Str("repo", repo.Name).Msg(err.Error())
sub.Fatal().Str("repo", repo.Name).Msg(err.Error())
}
err = testSSHConnection(site, sshAuth)
if err != nil {
log.Fatal().Str("stage", "locally").Str("repo", repo.Name).Msg(err.Error())
sub.Fatal().Str("repo", repo.Name).Msg(err.Error())
}
}
@ -451,6 +408,7 @@ func TempClone(repo types.Repo, tempdir string) (*git.Repository, error) {
}
func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url string) error {
sub = logger.CreateSubLogger("stage", "tempclone", "url", url)
token := destination.GetToken()
var auth transport.AuthMethod
if destination.SSH {
@ -462,17 +420,17 @@ func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url strin
err := site.GetValues(url)
if err != nil {
log.Fatal().Str("stage", "tempclone").Str("url", url).Msg(err.Error())
sub.Fatal().Msg(err.Error())
}
sshAuth, err := goph.Key(destination.SSHKey, "")
if err != nil {
log.Fatal().Str("stage", "tempclone").Str("url", url).Msg(err.Error())
sub.Fatal().Msg(err.Error())
}
err = testSSHConnection(site, sshAuth)
if err != nil {
log.Fatal().Str("stage", "tempclone").Str("url", url).Msg(err.Error())
sub.Fatal().Msg(err.Error())
}
if destination.SSHKey == "" {
home := os.Getenv("HOME")

15
logger/logger.go
View file

@ -46,3 +46,18 @@ func CreateLogger(conf types.Logging) zerolog.Logger {
return zerolog.New(mw).With().Timestamp().Logger()
}
// CreateSubLogger create a sublogger for modules
func CreateSubLogger(args ...string) zerolog.Logger {
sub := log.With()
for i := 0; i < len(args); i += 2 {
if i+1 < len(args) {
key := args[i]
value := args[i+1]
// Add key-value pairs to the sublogger
sub = sub.Str(key, value)
}
}
return sub.Logger()
}

6
main.go
View file

@ -237,7 +237,7 @@ func backup(repos []types.Repo, conf *types.Conf) {
Msg(err.Error())
continue
}
defer os.RemoveAll(tempdir)
temprepo, err := local.TempClone(r, tempdir)
if err != nil {
@ -315,7 +315,7 @@ func backup(repos []types.Repo, conf *types.Conf) {
Msg(err.Error())
continue
}
temprepo, err := local.TempClone(r, tempdir)
if err != nil {
if err == git.NoErrAlreadyUpToDate {
@ -393,7 +393,7 @@ func backup(repos []types.Repo, conf *types.Conf) {
Msg(err.Error())
continue
}
temprepo, err := local.TempClone(r, tempdir)
if err != nil {
if err == git.NoErrAlreadyUpToDate {

57
onedev/onedev.go
View file

@ -4,9 +4,14 @@ import (
"fmt"
"time"
"github.com/cooperspencer/gickup/logger"
"github.com/cooperspencer/gickup/types"
"github.com/cooperspencer/onedev"
"github.com/rs/zerolog/log"
"github.com/rs/zerolog"
)
var (
sub zerolog.Logger
)
func Get(conf *types.Conf) ([]types.Repo, bool) {
@ -18,20 +23,17 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
if repo.URL == "" {
repo.URL = "https://code.onedev.io/"
}
sub = logger.CreateSubLogger("stage", "onedev", "url", repo.URL)
err := repo.Filter.ParseDuration()
if err != nil {
log.Error().
Str("stage", "onedev").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
}
include := types.GetMap(repo.Include)
exclude := types.GetMap(repo.Exclude)
excludeorgs := types.GetMap(repo.ExcludeOrgs)
log.Info().
Str("stage", "onedev").
Str("url", repo.URL).
sub.Info().
Msgf("grabbing repositories from %s", repo.User)
if repo.Password == "" && repo.Token != "" {
@ -61,9 +63,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
if repo.User == "" {
u, _, err := client.GetMe()
if err != nil {
log.Error().
Str("stage", "onedev").
Str("url", repo.URL).
sub.Error().
Msg("can't find user")
break
}
@ -77,9 +77,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
userrepos, _, err := client.GetProjects(&query)
if err != nil {
log.Error().
Str("stage", "onedev").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
}
@ -100,18 +98,14 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
urls, _, err := client.GetCloneUrl(r.ID)
if err != nil {
log.Error().
Str("stage", "onedev").
Str("url", repo.URL).
sub.Error().
Msg("couldn't get clone urls")
continue
}
defaultbranch, _, err := client.GetDefaultBranch(r.ID)
if err != nil {
log.Error().
Str("stage", "onedev").
Str("url", repo.URL).
sub.Error().
Msgf("couldn't get default branch for %s", r.Name)
defaultbranch = "main"
}
@ -121,9 +115,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
if len(commits) > 0 {
commit, _, err := client.GetCommit(r.ID, commits[0])
if err != nil {
log.Error().
Str("stage", "onedev").
Str("url", repo.URL).
sub.Error().
Msgf("can't get latest commit for %s", defaultbranch)
} else {
lastactive := time.UnixMicro(commit.Author.When)
@ -149,18 +141,14 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
if repo.Username != "" && repo.Password != "" && len(repo.IncludeOrgs) == 0 && user.Name != "" {
memberships, _, err := client.GetUserMemberships(user.ID)
if err != nil {
log.Error().
Str("stage", "onedev").
Str("url", repo.URL).
sub.Error().
Msgf("couldn't get memberships for %s", user.Name)
}
for _, membership := range memberships {
group, _, err := client.GetGroup(membership.GroupID)
if err != nil {
log.Error().
Str("stage", "onedev").
Str("url", repo.URL).
sub.Error().
Msgf("couldn't get group with id %d", membership.GroupID)
}
if !excludeorgs[group.Name] {
@ -175,9 +163,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
orgrepos, _, err := client.GetProjects(&query)
if err != nil {
log.Error().
Str("stage", "onedev").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
}
@ -189,18 +175,14 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
}
urls, _, err := client.GetCloneUrl(r.ID)
if err != nil {
log.Error().
Str("stage", "onedev").
Str("url", repo.URL).
sub.Error().
Msg("couldn't get clone urls")
continue
}
defaultbranch, _, err := client.GetDefaultBranch(r.ID)
if err != nil {
log.Error().
Str("stage", "onedev").
Str("url", repo.URL).
sub.Error().
Msgf("couldn't get default branch for %s", r.Name)
defaultbranch = "main"
}
@ -229,6 +211,7 @@ func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) {
if destination.URL == "" {
destination.URL = "https://code.onedev.io/"
}
sub = logger.CreateSubLogger("stage", "onedev", "url", destination.URL)
if destination.Token != "" || destination.TokenFile != "" {
client = onedev.NewClient(destination.URL, onedev.SetToken(destination.GetToken()))

38
sourcehut/sourcehut.go
View file

@ -9,8 +9,13 @@ import (
"strings"
"time"
"github.com/cooperspencer/gickup/logger"
"github.com/cooperspencer/gickup/types"
"github.com/rs/zerolog/log"
"github.com/rs/zerolog"
)
var (
sub zerolog.Logger
)
// doRequest TODO
@ -137,11 +142,10 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
ran := false
repos := []types.Repo{}
for _, repo := range conf.Source.Sourcehut {
sub = logger.CreateSubLogger("stage", "sourcehut", "url", repo.URL)
err := repo.Filter.ParseDuration()
if err != nil {
log.Error().
Str("stage", "sourcehut").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
}
ran = true
@ -161,27 +165,21 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
user := User{}
body, err := doRequest(fmt.Sprintf("%suser", apiURL), token)
if err != nil {
log.Error().
Str("stage", "sourcehut").
Str("url", repo.URL).
sub.Error().
Msg("no user associated with this token")
continue
}
err = json.Unmarshal(body, &user)
if err != nil {
log.Error().
Str("stage", "sourcehut").
Str("url", repo.URL).
sub.Error().
Msg("cannot unmarshal user")
continue
}
repo.User = user.Name
}
log.Info().
Str("stage", "sourcehut").
Str("url", repo.URL).
sub.Info().
Msgf("grabbing repositories from %s", repo.User)
if repo.User != "" {
@ -197,9 +195,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
repositories, err := getRepos(apiURL, token)
if err != nil {
log.Error().
Str("stage", "sourcehut").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
}
@ -209,9 +205,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
refs, err := getRefs(apiURL, r.Name, token)
if err != nil {
log.Error().
Str("stage", "sourcehut").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
}
@ -225,9 +219,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
commits, err := getCommits(apiURL, r.Name, token)
if err != nil {
log.Error().
Str("stage", "sourcehut").
Str("url", repo.URL).
sub.Error().
Msg(err.Error())
} else {
if len(commits.Results) > 0 {
@ -325,6 +317,8 @@ func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) {
destination.URL = "https://git.sr.ht"
}
sub = logger.CreateSubLogger("stage", "sourcehut", "url", destination.URL)
if !strings.HasSuffix(destination.URL, "/") {
destination.URL += "/"
}