Skip to content

Commit 9714d6d

Browse files
authored
Various improvement (#123)
* clean: remove unneeded function Signed-off-by: Olivier Vernin <olivier.vernin@suse.com> * fix: golangci deprecation Signed-off-by: Olivier Vernin <olivier.vernin@suse.com> * chore: add comment Signed-off-by: Olivier Vernin <olivier.vernin@suse.com> * chore: rename function Signed-off-by: Olivier Vernin <olivier.vernin@suse.com> * deps: update golang dependencies Signed-off-by: Olivier Vernin <olivier.vernin@suse.com> * feat: allow to disable database migration Signed-off-by: Olivier Vernin <olivier.vernin@suse.com> * deps: bump postgresql in docker-compose.yaml Signed-off-by: Olivier Vernin <olivier.vernin@suse.com> * deps: bump updatecli policies Signed-off-by: Olivier Vernin <olivier.vernin@suse.com> * feat: return reports based on time instead of limit of 1000 Signed-off-by: Olivier Vernin <olivier.vernin@suse.com> * feat: allow to return scm summary information Signed-off-by: Olivier Vernin <olivier.vernin@suse.com> * feat: init Contributing.md Signed-off-by: Olivier Vernin <olivier.vernin@suse.com> * fix: git merge Signed-off-by: Olivier Vernin <olivier.vernin@suse.com> --------- Signed-off-by: Olivier Vernin <olivier.vernin@suse.com>
1 parent c8ff846 commit 9714d6d

8 files changed

Lines changed: 227 additions & 27 deletions

File tree

CONTRIBUTING.md

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# CONTIRUBUTING
2+
3+
The Udash project is a web application designed to better understand the state of Git repository.
4+
5+
It's composed of:
6+
* A Golang HTTP service (This repository)
7+
* A Vue.js Frontend [updatecli/udash-front](https://github.com/updatecli/udash-front)
8+
* Updatecli as the agent
9+
10+
The following instruction is the best way to start Udash within a dev environment.
11+
12+
## 1. Database
13+
14+
Udash relies on a postgresql database to store its data.
15+
We can spin up a local one running
16+
17+
* `make db.start`
18+
19+
Then we can test sql command running
20+
21+
* `PGPASSWORD=password psql --username=udash --file test.sql postgres://localhost:5432/udash `
22+
23+
## 2. Server
24+
25+
The server fetches its configuration from one of the following files (order matter):
26+
27+
1. "config.yaml"
28+
2. "$HOME/.udash/config.yaml"
29+
3. "/etc/udash/config.yaml"
30+
31+
```config.yaml
32+
server:
33+
auth:
34+
## Disable authentication
35+
mode: "none"
36+
## If dryrun is set to true, then only GET request are allowed
37+
dryrun: false
38+
39+
database:
40+
uri: "postgres://udash:password@localhost:5432/udash?sslmode=disable"
41+
```
42+
43+
* `make server.start`
44+
45+
### Endpoints
46+
47+
The file `pkg/server/main.go` contains the following endpoint:
48+
49+
* `/api/ping` [GET]
50+
* `/api/about`[GET]
51+
* `/api/pipeline/scms`[GET]
52+
* `/api/pipeline/reports`[GET][POST]
53+
* `/api/pipeline/reports/:id`[GET][PUT][DELETE]
54+
55+
## 3. Frontend
56+
57+
Information to start the frontend is available on [github.com/updatecli/udash-front](https://github.com/updatecli/udash-front)
58+
59+
60+
## 4. Agent
61+
62+
The Updatecli command is used to provide all the data to Udash.
63+
Please be aware that Udash is currently an experimental feature
64+
within Updatecli and must be enabled with the flag `--experimental`
65+
More information on `updatecli udash --help --experimental`

docker-compose.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
version: '3'
22
services:
33
db:
4-
image: postgres:15
4+
image: postgres:17@sha256:8d3be35b184e70d81e54cbcbd3df3c0b47f37d06482c0dd1c140db5dbcc6a808
55
restart: always
66
environment:
77
- POSTGRES_USER=udash

pkg/database/main.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@ var (
3131

3232
type Options struct {
3333
// URI defines the DB URI
34-
URI string
34+
URI string
35+
MigrationDisabled bool
3536
}
3637

3738
func Connect(o Options) error {

pkg/engine/main.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,11 @@ func (e *Engine) Start() {
2828
os.Exit(1)
2929
}
3030

31-
if err := database.RunMigrationUp(); err != nil {
32-
logrus.Errorln(err)
33-
os.Exit(1)
31+
if !e.Options.Database.MigrationDisabled {
32+
if err := database.RunMigrationUp(); err != nil {
33+
logrus.Errorln(err)
34+
os.Exit(1)
35+
}
3436
}
3537

3638
s := server.Server{

pkg/server/reportHandler.go

Lines changed: 25 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -92,20 +92,26 @@ func FindAllPipelineReports(c *gin.Context) {
9292
switch scmid {
9393
case "":
9494
query = `
95-
SELECT id, data, created_at, updated_at
96-
FROM pipelineReports
97-
ORDER BY updated_at DESC FETCH FIRST 1000 ROWS ONLY`
95+
SELECT id, data, created_at, updated_at
96+
FROM pipelineReports
97+
WHERE updated_at > current_date - interval '%d day'
98+
ORDER BY updated_at DESC`
99+
query = fmt.Sprintf(query, monitoringDurationDays)
98100

99101
case "none", "null", "nil":
100102
query = `
101-
SELECT DISTINCT ON (s.data ->> 'Name') s.id, s.data, s.created_at, s.updated_at
102-
FROM (
103-
SELECT *
104-
FROM pipelinereports
105-
WHERE NOT jsonb_path_exists(data::jsonb, '$.Targets[*].* ? (@.Scm.URL != "" && @.Scm.Branch.Target != "")')
106-
ORDER BY updated_at DESC
107-
) s
108-
ORDER BY s.data ->> 'Name', s.updated_at DESC FETCH FIRST 1000 ROWS ONLY;`
103+
SELECT DISTINCT ON (s.data ->> 'Name') s.id, s.data, s.created_at, s.updated_at
104+
FROM (
105+
SELECT *
106+
FROM pipelinereports
107+
WHERE
108+
NOT jsonb_path_exists(data::jsonb, '$.Targets[*].* ? (@.Scm.URL != "" && @.Scm.Branch.Target != "")') AND
109+
updated_at > current_date - interval '%d day'
110+
ORDER BY updated_at DESC
111+
) s
112+
ORDER BY s.data ->> 'Name', s.updated_at DESC;`
113+
114+
query = fmt.Sprintf(query, monitoringDurationDays)
109115

110116
default:
111117
scm, err := dbGetScm(scmid, "", "")
@@ -122,15 +128,17 @@ func FindAllPipelineReports(c *gin.Context) {
122128
query = `
123129
SELECT DISTINCT ON (s.data ->> 'Name') s.id, s.data, s.created_at, s.updated_at
124130
FROM (
125-
SELECT *
126-
FROM pipelinereports
127-
WHERE jsonb_path_exists(data::jsonb, '$.Targets[*].* ? (@.Scm.URL == "%s" && @.Scm.Branch.Target == "%s")')
128-
ORDER BY updated_at DESC
131+
SELECT *
132+
FROM pipelinereports
133+
WHERE
134+
jsonb_path_exists(data::jsonb, '$.Targets[*].* ? (@.Scm.URL == "%s" && @.Scm.Branch.Target == "%s")') AND
135+
updated_at > current_date - interval '%d day'
136+
ORDER BY updated_at DESC
129137
) s
130-
ORDER BY s.data ->> 'Name', s.updated_at DESC FETCH FIRST 1000 ROWS ONLY;
138+
ORDER BY s.data ->> 'Name', s.updated_at DESC;
131139
`
132140

133-
query = fmt.Sprintf(query, scm[0].URL, scm[0].Branch)
141+
query = fmt.Sprintf(query, scm[0].URL, scm[0].Branch, monitoringDurationDays)
134142

135143
default:
136144
// Normally we should never have multiple scms with the same id

pkg/server/scmDBHandler.go

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,22 @@
11
package server
22

33
import (
4+
"context"
5+
"fmt"
46
"net/http"
7+
"strings"
58

69
"github.com/gin-gonic/gin"
710
"github.com/sirupsen/logrus"
11+
"github.com/updatecli/udash/pkg/database"
812
)
913

1014
func FindSCM(c *gin.Context) {
1115

1216
scmid := c.Request.URL.Query().Get("scmid")
1317
url := c.Request.URL.Query().Get("url")
1418
branch := c.Request.URL.Query().Get("branch")
19+
summary := c.Request.URL.Query().Get("summary")
1520

1621
rows, err := dbGetScm(scmid, url, branch)
1722
if err != nil {
@@ -23,7 +28,120 @@ func FindSCM(c *gin.Context) {
2328
return
2429
}
2530

31+
if strings.ToUpper(summary) == "TRUE" {
32+
FindSCMSummary(c, rows)
33+
return
34+
}
35+
2636
c.JSON(http.StatusOK, gin.H{
2737
"scms": rows,
2838
})
2939
}
40+
41+
// FindSCMSummary returns a summary of all git repositories detected.
42+
func FindSCMSummary(c *gin.Context, scmRows []DatabaseSCMRow) {
43+
44+
type scmSummaryData struct {
45+
ID string `json:"id"`
46+
TotalResultByType map[string]int `json:"total_result_by_type"`
47+
TotalResult int `json:"total_result"`
48+
}
49+
50+
type scmBranchData map[string]scmSummaryData
51+
52+
type data struct {
53+
// URLS is a map of scmURLS where the key is the scm URL.
54+
Data map[string]scmBranchData
55+
}
56+
57+
dataset := data{}
58+
59+
query := ""
60+
for _, row := range scmRows {
61+
62+
scmID := row.ID
63+
scmURL := row.URL
64+
scmBranch := row.Branch
65+
66+
if scmBranch == "" || scmURL == "" {
67+
logrus.Debugf("skipping scm %s, missing branch or url", row.ID)
68+
continue
69+
}
70+
71+
query = `
72+
SELECT DISTINCT ON (s.data ->> 'Name') s.id, ( s.data ->> 'Result')
73+
FROM (
74+
SELECT *
75+
FROM pipelinereports
76+
WHERE
77+
jsonb_path_exists(data::jsonb, '$.Targets[*].* ? (@.Scm.URL == "%s" && @.Scm.Branch.Target == "%s")') AND
78+
updated_at > current_date - interval '%d day'
79+
ORDER BY updated_at DESC
80+
) s
81+
ORDER BY s.data ->> 'Name', s.updated_at DESC;
82+
`
83+
84+
query = fmt.Sprintf(query, scmURL, scmBranch, monitoringDurationDays)
85+
86+
rows, err := database.DB.Query(context.Background(), query)
87+
if err != nil {
88+
logrus.Errorf("query failed: %s", err)
89+
c.JSON(http.StatusInternalServerError, gin.H{
90+
"message": err,
91+
})
92+
return
93+
}
94+
95+
if dataset.Data == nil {
96+
dataset.Data = make(map[string]scmBranchData)
97+
}
98+
99+
if dataset.Data[scmURL] == nil {
100+
dataset.Data[scmURL] = make(map[string]scmSummaryData)
101+
}
102+
103+
d := scmSummaryData{
104+
ID: scmID.String(),
105+
TotalResultByType: make(map[string]int),
106+
}
107+
108+
dataset.Data[scmURL][scmBranch] = d
109+
110+
for rows.Next() {
111+
112+
id := ""
113+
result := ""
114+
115+
err = rows.Scan(&id, &result)
116+
if err != nil {
117+
logrus.Errorf("parsing result: %s", err)
118+
c.JSON(http.StatusInternalServerError, gin.H{
119+
"message": err,
120+
})
121+
return
122+
}
123+
124+
resultFound := false
125+
for r := range dataset.Data[scmURL][scmBranch].TotalResultByType {
126+
if r == result {
127+
dataset.Data[scmURL][scmBranch].TotalResultByType[r]++
128+
resultFound = true
129+
}
130+
}
131+
132+
if !resultFound {
133+
dataset.Data[scmURL][scmBranch].TotalResultByType[result] = 1
134+
}
135+
}
136+
137+
scmData := dataset.Data[scmURL][scmBranch]
138+
for r := range scmData.TotalResultByType {
139+
scmData.TotalResult += scmData.TotalResultByType[r]
140+
}
141+
dataset.Data[scmURL][scmBranch] = scmData
142+
}
143+
144+
c.JSON(http.StatusOK, gin.H{
145+
"data": dataset.Data,
146+
})
147+
}

pkg/server/var.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
package server
2+
3+
var (
4+
// monitoringDurationDays is the default number of days to search for reports.
5+
monitoringDurationDays int = 30
6+
)

update-compose.yaml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,24 +3,24 @@ policies:
33
# config:
44
# - updatecli/updatecli.d/
55
- name: Update Updatecli policies
6-
policy: ghcr.io/updatecli/policies/updatecli/autodiscovery:0.2.0@sha256:46e599fb7e874ee3f1c9cf5e4dd64b328a15d61d03e106176b19a07683afde29
6+
policy: ghcr.io/updatecli/policies/updatecli/autodiscovery:0.5.0@sha256:947817644fb89e27f7b7121b822328c2d47364c7a3a08241e4d2ac1a5897020c
77
values:
88
- updatecli/values.d/scm.yaml
99
- name: Golang Version
10-
policy: ghcr.io/updatecli/policies/golang/version:0.1.0@sha256:8bdbd911916c45288b85e7437ef8e140966321177128ca2460d21a2e5c7eedd8
10+
policy: ghcr.io/updatecli/policies/golang/version:0.4.0@sha256:9b3c09a73ffbecc690f07a4a44244f51dcc1dfb0e5a292207f2543b79f08fcaa
1111
values:
1212
- updatecli/values.d/scm.yaml
1313
- name: Minor Golang Module update
14-
policy: ghcr.io/updatecli/policies/autodiscovery/golang:0.7.0@sha256:a9d7947b2e0ad241f02beb3900d285b51b8336e3cf619118a9b37f1fe97f82ed
14+
policy: ghcr.io/updatecli/policies/autodiscovery/golang:0.10.0@sha256:3b1a2b03b3cd8e33305ca165a8be2ca126a06be4cbdf388bf98e48a568855cdc
1515
values:
1616
- updatecli/values.d/scm.yaml
1717
- updatecli/values.d/golang_minor.yaml
1818
- name: Patch Golang Module update
19-
policy: ghcr.io/updatecli/policies/autodiscovery/golang:0.7.0@sha256:a9d7947b2e0ad241f02beb3900d285b51b8336e3cf619118a9b37f1fe97f82ed
19+
policy: ghcr.io/updatecli/policies/autodiscovery/golang:0.10.0@sha256:3b1a2b03b3cd8e33305ca165a8be2ca126a06be4cbdf388bf98e48a568855cdc
2020
values:
2121
- updatecli/values.d/scm.yaml
2222
- updatecli/values.d/golang_patch.yaml
2323
- name: Update golangci-lint
24-
policy: ghcr.io/updatecli/policies/golangci-lint/githubaction:0.1.0@sha256:099fe49e355afe82b36d5ecc8cb30014fd2938700b55aa0fdfce7169c77d3071
24+
policy: ghcr.io/updatecli/policies/golangci-lint/githubaction:0.5.0@sha256:0e1addae151528a2c337a306eb323332dd36a91f36b0e93891858eed7466456b
2525
values:
2626
- updatecli/values.d/scm.yaml

0 commit comments

Comments
 (0)