Add sync files struct

Author: JojiiOfficial

SyncFiles.go

@@ -0,0 +1,143 @@
+package main
+
+import (
+	"bufio"
+	"database/sql"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"sync"
+
+	"github.com/JojiiOfficial/gaw"
+)
+
+func syncFiles(dryRun, noconfirm bool) error {
+	// List locally available files
+	files, err := ioutil.ReadDir(config.Server.PathConfig.FileStore)
+	if err != nil {
+		return err
+	}
+
+	// Get Local files+ID
+	rows, err := db.Table("files").Where("deleted_at is NULL").Select("id, local_name").Rows()
+	if err != nil {
+		return err
+	}
+
+	var untrackedInFS []int64  // Database file rows without local file
+	var untrackedInDB []string // Local files which don't have any reference in database
+	var wg sync.WaitGroup
+
+	wg.Add(2)
+	cerr := make(chan error, 2)
+
+	go func() {
+		untrackedInFS, err = searchLocalFS(rows, files)
+		cerr <- err
+		wg.Done()
+	}()
+
+	go func() {
+		untrackedInDB, err = searchDB(files)
+		cerr <- err
+		wg.Done()
+	}()
+
+	wg.Wait()
+	close(cerr)
+
+	for i := range cerr {
+		if i != nil {
+			return i
+		}
+	}
+
+	if len(untrackedInDB) == 0 && len(untrackedInFS) == 0 {
+		fmt.Println("Nothing to do")
+		return nil
+	}
+
+	fmt.Println()
+	fmt.Printf("Found %d files missing in local filesystem\n", len(untrackedInFS))
+	fmt.Printf("Found %d files untracked in database\n", len(untrackedInDB))
+
+	if dryRun {
+		return nil
+	}
+
+	if !noconfirm {
+		if y, _ := gaw.ConfirmInput("\nDelete all untracked files? [y/n/a]> ", bufio.NewReader(os.Stdin)); !y {
+			return nil
+		}
+	}
+
+	fmt.Println("Deleting...")
+
+	return nil
+}
+
+// Returns a slice of FileIDs which have no local file
+func searchLocalFS(rows *sql.Rows, files []os.FileInfo) ([]int64, error) {
+	var untrackedInFS []int64
+
+	var fileName string
+	var id int64
+	for rows.Next() {
+		if err := rows.Scan(&id, &fileName); err != nil {
+			return nil, err
+		}
+
+		if !hasLocalFile(files, fileName) {
+			untrackedInFS = append(untrackedInFS, id)
+		}
+	}
+
+	return untrackedInFS, nil
+}
+
+// Returns a slice of local files which don't have a reference in DB
+func searchDB(localFiles []os.FileInfo) ([]string, error) {
+	var untrackedInDB []string
+	var i, j, end int
+
+	for {
+		if i >= len(localFiles) {
+			break
+		}
+		end = i + 1000000
+
+		if end > len(localFiles) {
+			end = len(localFiles)
+		}
+
+		var res []string
+		fmt.Println(i, end)
+		if err := db.Table("files").Where("local_name NOT IN (?) AND deleted_at IS NULL", fifoToStr(localFiles[i:end])).Select("local_name").Find(&res).Error; err != nil {
+			return nil, err
+		}
+
+		untrackedInDB = append(untrackedInDB, res...)
+		i = end + 1
+		j++
+	}
+
+	return untrackedInDB, nil
+}
+
+func fifoToStr(f []os.FileInfo) []string {
+	s := make([]string, len(f))
+	for i := range f {
+		s[i] = f[i].Name()
+	}
+	return s
+}
+
+func hasLocalFile(localFiles []os.FileInfo, name string) bool {
+	for i := range localFiles {
+		if localFiles[i].Name() == name {
+			return true
+		}
+	}
+
+	return false
+}

go.mod

@@ -14,15 +14,16 @@ require (
 	github.com/gabriel-vasile/mimetype v1.1.1
 	github.com/gorilla/mux v1.7.4
 	github.com/h2non/filetype v1.1.0
-	github.com/jackc/pgx/v4 v4.7.2 // indirect
+	github.com/jackc/pgx/v4 v4.8.0 // indirect
+	github.com/jinzhu/gorm v1.9.15 // indirect
 	github.com/lib/pq v1.7.0 // indirect
 	github.com/mattn/go-colorable v0.1.7 // indirect
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect
 	github.com/sbani/go-humanizer v0.3.1
 	github.com/sirupsen/logrus v1.6.0
 	golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 // indirect
-	golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect
+	golang.org/x/sys v0.0.0-20200727154430-2d971f7391a4 // indirect
 	gopkg.in/yaml.v2 v2.3.0 // indirect
-	gorm.io/driver/postgres v0.2.5
-	gorm.io/gorm v0.2.22
+	gorm.io/driver/postgres v0.2.6
+	gorm.io/gorm v0.2.24
 )

go.sum

@@ -59,8 +59,8 @@ github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr
 github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI=
 github.com/jackc/pgconn v1.6.1 h1:lwofaXKPbIx6qEaK8mNm7uZuOwxHw+PnAFGDsDFpkRI=
 github.com/jackc/pgconn v1.6.1/go.mod h1:g8mKMqmSUO6AzAvha7vy07g1rbGOlc7iF0nU0ei83hc=
-github.com/jackc/pgconn v1.6.2 h1:ifRs/oHByR6NfEXfusvjoTqX/KcSvDYNFASoK/wXKfs=
-github.com/jackc/pgconn v1.6.2/go.mod h1:w2pne1C2tZgP+TvjqLpOigGzNqjBgQW9dUw/4Chex78=
+github.com/jackc/pgconn v1.6.3 h1:4Ks3RKvSvKPolXZsnLQTDAsokDhgID14Cv4ehECmzlY=
+github.com/jackc/pgconn v1.6.3/go.mod h1:w2pne1C2tZgP+TvjqLpOigGzNqjBgQW9dUw/4Chex78=
 github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
 github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
 github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2 h1:JVX6jT/XfzNqIjye4717ITLaNwV9mWbJx0dLCpcRzdA=
@@ -88,8 +88,8 @@ github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkAL
 github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ=
 github.com/jackc/pgtype v1.4.0 h1:pHQfb4jh9iKqHyxPthq1fr+0HwSNIl3btYPbw2m2lbM=
 github.com/jackc/pgtype v1.4.0/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig=
-github.com/jackc/pgtype v1.4.1 h1:8PRKqCS9Nt2FQbNegoEAIlY6r/DTP2aaXyh5bAEn89g=
-github.com/jackc/pgtype v1.4.1/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig=
+github.com/jackc/pgtype v1.4.2 h1:t+6LWm5eWPLX1H5Se702JSBcirq6uWa4jiG4wV1rAWY=
+github.com/jackc/pgtype v1.4.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig=
 github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y=
 github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM=
 github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc=
@@ -98,14 +98,16 @@ github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6
 github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg=
 github.com/jackc/pgx/v4 v4.7.1 h1:aqUSOcStk6fik+lSE+tqfFhvt/EwT8q/oMtJbP9CjXI=
 github.com/jackc/pgx/v4 v4.7.1/go.mod h1:nu42q3aPjuC1M0Nak4bnoprKlXPINqopEKqbq5AZSC4=
-github.com/jackc/pgx/v4 v4.7.2 h1:0DJC1AiqH0Lba79JHFQkcoxi0sOAn75Zr+QCRCAXvBc=
-github.com/jackc/pgx/v4 v4.7.2/go.mod h1:IaoCMFiHwe2J7SjRZ97Qc7zr8QGNwnlAU4J0f3S1UYk=
+github.com/jackc/pgx/v4 v4.8.0 h1:xO3bPvwr0MJSoDfb4yeeWZIxSZ2VFBm5axPnaNEnGUQ=
+github.com/jackc/pgx/v4 v4.8.0/go.mod h1:AjqYcDmEyst6GF8jJi/RF73Gla9d7/HLZzJEZj2uwpM=
 github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jinzhu/gorm v1.9.14 h1:Kg3ShyTPcM6nzVo148fRrcMO6MNKuqtOUwnzqMgVniM=
 github.com/jinzhu/gorm v1.9.14/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs=
+github.com/jinzhu/gorm v1.9.15 h1:OdR1qFvtXktlxk73XFYMiYn9ywzTwytqe4QkuMRqc38=
+github.com/jinzhu/gorm v1.9.15/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M=
@@ -225,8 +227,8 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
-golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200727154430-2d971f7391a4 h1:gtF+PUC1CD1a9ocwQHbVNXuTp6RQsAYt6tpi6zjT81Y=
+golang.org/x/sys v0.0.0-20200727154430-2d971f7391a4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
@@ -254,9 +256,9 @@ gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gorm.io/driver/postgres v0.2.5 h1:HAzk6g9BhPtaJOcdf7WFaR7VJStENs8Ksi6WvIWyj/c=
-gorm.io/driver/postgres v0.2.5/go.mod h1:AsPyuhKFOplSmQwOPsycVKbe0dRxF8v18KZ7p9i8dIs=
+gorm.io/driver/postgres v0.2.6 h1:hoE6SzA5wKOo6AYxz2V7ooxnzD6S6ToLAHHDDawt+b0=
+gorm.io/driver/postgres v0.2.6/go.mod h1:AsPyuhKFOplSmQwOPsycVKbe0dRxF8v18KZ7p9i8dIs=
 gorm.io/gorm v0.2.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
-gorm.io/gorm v0.2.22 h1:kktbaUjQf4aA6L9o/qCghKRNuCvaWJycSToqvhQAB8c=
-gorm.io/gorm v0.2.22/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
+gorm.io/gorm v0.2.24 h1:NKjfopoXim5xAHYkt/uN0n7ZNmyoXZ9FEZUX4ijzXlI=
+gorm.io/gorm v0.2.24/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=

main.go

@@ -21,10 +21,12 @@ import (
 const version = "v3.12.0"
 
 var (
-	app         = kingpin.New("dmserver", "The data manager server")
-	appLogLevel = app.Flag("log-level", "Enable debug mode").HintOptions(constants.LogLevels...).Envar(getEnVar(EnVarLogLevel)).Short('l').Default(constants.LogLevels[2]).String()
-	appNoColor  = app.Flag("no-color", "Disable colors").Envar(getEnVar(EnVarNoColor)).Bool()
-	appCfgFile  = app.
+	app          = kingpin.New("dmserver", "The data manager server")
+	appLogLevel  = app.Flag("log-level", "Enable debug mode").HintOptions(constants.LogLevels...).Envar(getEnVar(EnVarLogLevel)).Short('l').Default(constants.LogLevels[2]).String()
+	appNoColor   = app.Flag("no-color", "Disable colors").Envar(getEnVar(EnVarNoColor)).Bool()
+	appNoConfirm = app.Flag("no-confirm", "Disable confirmations. Make it headless").Bool()
+	appDryRun    = app.Flag("dry-run", "Don't perform any action").Short('d').Bool()
+	appCfgFile   = app.
 			Flag("config", "the configuration file for the server").
 			Envar(getEnVar(EnVarConfigFile)).
 			Short('c').String()
@@ -39,6 +41,8 @@ var (
 	configCmd           = app.Command("config", "Commands for the config file")
 	configCmdCreate     = configCmd.Command("create", "Create config file")
 	configCmdCreateName = configCmdCreate.Arg("name", "Config filename").Default(models.GetDefaultConfig()).String()
+
+	syncFilesCmd = app.Command("sync-files", "Delete untracked files from the database and the filesystem")
 )
 
 var (
@@ -134,6 +138,15 @@ func main() {
 		{
 			startAPI()
 		}
+
+	// Tools
+	case syncFilesCmd.FullCommand():
+		{
+			if err := syncFiles(*appDryRun, *appNoConfirm); err != nil {
+				log.Error(err)
+			}
+		}
+
 	// Config --------------------
 	case configCmdCreate.FullCommand():
 		{

storage/Database.go

@@ -22,7 +22,7 @@ func ConnectToDatabase(config *models.Config) (*gorm.DB, error) {
 	}
 
 	//Automigration
-	db.AutoMigrate(
+	err = db.AutoMigrate(
 		&models.Role{},
 		&models.Namespace{},
 		&models.Tag{},