Empty dirs support

Author: colin969

downloader.go

@@ -2,15 +2,18 @@ package main
 
 import (
 	"context"
-	"crypto/sha1"
 	"database/sql"
+	"encoding/binary"
 	"encoding/hex"
 	"fmt"
 	"fyne.io/fyne/v2/dialog"
 	"github.com/cavaliergopher/grab/v3"
+	"hash/crc32"
+	"os"
 	"path/filepath"
 	"sync"
 	"time"
+	"unsafe"
 )
 
 type Update struct {
@@ -157,6 +160,34 @@ func (d *Downloader) Resume() error {
 		d.responderWg.Done()
 	}()
 
+	// Set up empty dirs handler
+
+	// Attach to any wait group, it's independent anyway
+	d.responderWg.Add(1)
+	go func() {
+		defer d.responderWg.Done()
+		// Loop finding empty dirs until done or context ends
+		for {
+			select {
+			case <-d.ctx.Done():
+				return
+			default:
+				// Get next empty dir
+				dir, err := d.state.Repo.GetNextEmptyDir()
+				if err != nil {
+					dialog.NewError(&DatabaseError{err}, d.state.window).Show()
+					return
+				}
+				dest := filepath.Join(d.installPath, dir)
+				err = os.MkdirAll(dest, os.ModePerm)
+				if err != nil {
+					dialog.NewError(&FatalDownloadFailure{err}, d.state.window).Show()
+					return
+				}
+			}
+		}
+	}()
+
 	// Set up UI updater
 	d.updaterWg.Add(1)
 	go func() {
@@ -428,11 +459,11 @@ func (d *Downloader) NewRequest(f *IndexedFile) (*grab.Request, error) {
 	}
 
 	// Add automatic checksumming
-	sum, err := hex.DecodeString(f.SHA1)
+	sum, err := hex.DecodeString(fmt.Sprintf("%08x", f.CRC32))
 	if err != nil {
 		return nil, err
 	}
-	req.SetChecksum(sha1.New(), sum, true)
+	req.SetChecksum(crc32.NewIEEE(), sum, true)
 	req.Size = f.Size
 	req = req.WithContext(d.ctx)
 	req.BufferSize = d.bufferSize
@@ -445,3 +476,9 @@ func (d *Downloader) NewRequest(f *IndexedFile) (*grab.Request, error) {
 
 	return req, nil
 }
+
+func intToBytes(n int) []byte {
+	bytes := make([]byte, unsafe.Sizeof(n)) // Assuming int is 4 bytes on your platform
+	binary.BigEndian.PutUint32(bytes, uint32(n))
+	return bytes
+}

index.py

@@ -2,7 +2,7 @@
 from tqdm import tqdm
 import os
 import sys
-import hashlib
+import zlib
 import posixpath
 import sqlite3
 
@@ -23,13 +23,13 @@ def win_path(path):
 
 
 def hash(file, bufsize=2 ** 16):
-    h = hashlib.new('sha1')
+    crc32 = 0
     with open(file, 'rb') as f:
         buf = f.read(bufsize)
         while len(buf) > 0:
-            h.update(buf)
+            crc32 = zlib.crc32(buf, crc32)
             buf = f.read(bufsize)
-    return h.hexdigest()
+    return crc32
 
 
 def index(path, name, base_url, db_file):
@@ -38,12 +38,12 @@ def index(path, name, base_url, db_file):
         os.remove(db_file)
 
     # Create Database
-    conn = sqlite3.connect(db_file)
+    conn = sqlite3.connect(":memory:")
     cur = conn.cursor()
 
     # Set up database
     insert_empty_dir_query = "INSERT INTO empty_dirs (path) VALUES (?)"
-    insert_file_query = "INSERT INTO files (path, size, sha1) VALUES (?, ?, ?)"
+    insert_file_query = "INSERT INTO files (path, size, crc32) VALUES (?, ?, ?)"
     overview_schema = """
     CREATE TABLE overview (
         name TEXT PRIMARY KEY,
@@ -56,7 +56,7 @@ def index(path, name, base_url, db_file):
     CREATE TABLE files (
         path TEXT PRIMARY KEY,
         size INTEGER,
-        sha1 INTEGER,
+        crc32 INTEGER,
         taken INTEGER DEFAULT false,
         done INTEGER DEFAULT false
     );
@@ -68,7 +68,7 @@ def index(path, name, base_url, db_file):
     );
     """
     index_statement = """
-    CREATE INDEX sha_idx ON files (sha1);
+    CREATE INDEX crc_idx ON files (crc32);
     """
     index2_statement = """
     CREATE INDEX done_idx ON files (done, taken);
@@ -95,15 +95,27 @@ def index(path, name, base_url, db_file):
                     size = os.path.getsize(f)
                     entries.append((x, size, hash(f)))
                     pbar.update(1)
-            if len(entries) > 100:
+            if len(entries) > 5000:
                 cur.executemany(insert_file_query, entries)
+                conn.commit()
                 entries.clear()
+    if len(entries) > 0:
+        cur.executemany(insert_file_query, entries)
+        conn.commit()
+        entries.clear()
     # Get overview info
     cur.execute("SELECT SUM(size), COUNT(*) FROM files;")
     res = cur.fetchone()
     cur.execute("INSERT INTO overview (name, total_size, total_files, base_url) VALUES (?,?,?,?);",
                 (name, res[0], res[1], base_url))
     conn.commit()
+
+    print("Saving Database...")
+    with sqlite3.connect(db_file) as disk_db:
+        disk_db.executescript("".join(conn.iterdump()))
+
+    print("Done!")
+
     return
 
 

main.go

@@ -172,19 +172,22 @@ func setupLayout(w fyne.Window, state *InstallerState) *fyne.Container {
 
 func mainLayout(w fyne.Window, state *InstallerState) *fyne.Container {
 	pathHeaderLabel := widget.NewLabel("Install Path: ")
+	pathHeaderLabel.TextStyle = fyne.TextStyle{Bold: true}
 	pathLabel := widget.NewLabelWithData(state.folderPath)
 	pathContainer := container.New(layout.NewHBoxLayout(),
 		pathHeaderLabel,
 		pathLabel)
 
 	sourceHeaderLabel := widget.NewLabel("Source: ")
+	sourceHeaderLabel.TextStyle = fyne.TextStyle{Bold: true}
 	sourceLabel := widget.NewLabel(state.baseUrl)
 	sourceContainer := container.New(layout.NewHBoxLayout(),
 		sourceHeaderLabel,
 		sourceLabel)
 
 	// Create active file bars
 	fileLabel := widget.NewLabel("File: ")
+	fileLabel.TextStyle = fyne.TextStyle{Bold: true}
 	fileHeader1 := widget.NewLabelWithData(state.fileTitle1)
 	fileHeader1.Alignment = fyne.TextAlignLeading
 	fileHeader1.Wrapping = fyne.TextTruncate
@@ -212,7 +215,6 @@ func mainLayout(w fyne.Window, state *InstallerState) *fyne.Container {
 	progressBarTotal := widget.NewProgressBarWithData(state.progressBarTotal)
 	totalLabel := canvas.NewText("Total Progress...", color.White)
 
-	rateLimitLabel := widget.NewLabel("Download Speed Limit:")
 	rateLimitCurrentLabel := widget.NewLabelWithData(state.formatRateLimit)
 	rateLimitEntry := widget.NewEntryWithData(state.rateLimitEntry)
 	rateLimitSet := widget.NewButton("Set (KB/s)", func() {
@@ -260,11 +262,7 @@ func mainLayout(w fyne.Window, state *InstallerState) *fyne.Container {
 		}
 	})
 
-	rateLimContainer := container.New(layout.NewVBoxLayout(),
-		container.New(layout.NewHBoxLayout(),
-			rateLimitLabel,
-			rateLimitCurrentLabel),
-		container.NewBorder(nil, nil, nil, rateLimitSet, rateLimitEntry))
+	rateLimContainer := container.NewBorder(nil, nil, nil, rateLimitSet, rateLimitEntry)
 
 	// Create buttons
 	button1 := widget.NewButton("Start", func() {
@@ -306,8 +304,7 @@ func mainLayout(w fyne.Window, state *InstallerState) *fyne.Container {
 	// Create a row with buttons
 	buttonsRow := container.NewHBox(button1, button2, button3)
 
-	// Create 3 additional labels
-	downloadedHeaderLabel := widget.NewLabel("Downloaded:")
+	// Create stats labels
 	downloadedLabel := widget.NewLabelWithData(state.formatDownloadedSize)
 	downloadedLabel.Alignment = fyne.TextAlignLeading
 	downloadedLabel.TextStyle = fyne.TextStyle{Monospace: true}
@@ -319,23 +316,30 @@ func mainLayout(w fyne.Window, state *InstallerState) *fyne.Container {
 		widget.NewLabel("/"),
 		totalSizeLabel)
 
-	speedHeaderLabel := widget.NewLabel("Average Speed:")
-	speedHeaderLabel.Alignment = fyne.TextAlignLeading
 	speedLabel := widget.NewLabelWithData(state.formatDownloadSpeed)
 	speedLabel.Alignment = fyne.TextAlignLeading
 	speedLabel.TextStyle = fyne.TextStyle{Monospace: true}
 
+	statsForm := &widget.Form{
+		Items: []*widget.FormItem{
+			{Text: "Downloaded:", Widget: downloadedContainer},
+			{Text: "Average Speed:", Widget: speedLabel},
+			{Text: "Download Speed Limit:", Widget: rateLimitCurrentLabel},
+		},
+	}
+
 	leftMainContent := container.New(layout.NewVBoxLayout(),
 		pathContainer,
-		sourceContainer,
 		totalLabel,
 		progressBarTotal,
+		statsForm,
 		layout.NewSpacer(),
 		rateLimContainer,
 		buttonsRow,
 	)
 
 	rightMainContent := container.New(layout.NewVBoxLayout(),
+		sourceContainer,
 		fileContainer1,
 		fileProgressBar1,
 		fileContainer2,
@@ -347,21 +351,8 @@ func mainLayout(w fyne.Window, state *InstallerState) *fyne.Container {
 
 	mainContent := container.NewBorder(nil, nil, leftMainContent, nil, rightMainContent)
 
-	line := canvas.NewLine(color.Gray{Y: 0x55})
-	line.StrokeWidth = 2
-
-	leftSideContent := container.New(layout.NewHBoxLayout(),
-		container.New(layout.NewVBoxLayout(),
-			layout.NewSpacer(),
-			downloadedHeaderLabel,
-			downloadedContainer,
-			speedHeaderLabel,
-			speedLabel),
-		line,
-	)
-
 	// Combine the grid layout and sidebar label in a horizontal box
-	mainLayout := container.NewBorder(topBarLayout("install"), nil, leftSideContent, nil, mainContent)
+	mainLayout := container.NewBorder(topBarLayout("install"), nil, nil, nil, mainContent)
 
 	return mainLayout
 }
@@ -461,7 +452,7 @@ func FormatBytes(size int64) string {
 
 	var unitIndex int
 	floatSize := float64(size)
-	for unitIndex = 0; unitIndex < len(units); unitIndex++ {
+	for unitIndex = 0; unitIndex < len(units)-1; unitIndex++ {
 		if floatSize < 1024.0 {
 			break
 		}

repo.go

@@ -52,6 +52,20 @@ func (repo *SqliteRepo) MarkFileDone(file *IndexedFile) error {
 	return err
 }
 
+func (repo *SqliteRepo) GetNextEmptyDir() (string, error) {
+	var d string
+	err := repo.db.QueryRow(`UPDATE empty_dirs SET done = true
+		WHERE rowid = (
+		    SELECT MIN(rowid)
+		    FROM empty_dirs
+		    WHERE done = false
+		) RETURNING path`).Scan(&d)
+	if err != nil {
+		return "", err
+	}
+	return d, err
+}
+
 func (repo *SqliteRepo) GetNextFile() (*IndexedFile, error) {
 	var f IndexedFile
 	f.RetryCount = 0
@@ -60,15 +74,16 @@ func (repo *SqliteRepo) GetNextFile() (*IndexedFile, error) {
 		    SELECT MIN(rowid)
 		    FROM files
 		    WHERE done = false AND taken = false
-		) RETURNING path, size, sha1`).Scan(&f.Filepath, &f.Size, &f.SHA1)
+		) RETURNING path, size, crc32`).Scan(&f.Filepath, &f.Size, &f.CRC32)
 	if err != nil {
 		return nil, err
 	}
 	return &f, nil
 }
 
+// GetNextFileBatch Cannot be safely executed in parallel
 func (repo *SqliteRepo) GetNextFileBatch(limit int64) ([]*IndexedFile, error) {
-	rows, err := repo.db.Query("SELECT path, size, sha1 FROM files WHERE done = false AND taken = false LIMIT ?", limit)
+	rows, err := repo.db.Query("SELECT path, size, crc32 FROM files WHERE done = false AND taken = false LIMIT ?", limit)
 	if err != nil {
 		return nil, err
 	}
@@ -78,7 +93,7 @@ func (repo *SqliteRepo) GetNextFileBatch(limit int64) ([]*IndexedFile, error) {
 	for rows.Next() {
 		var f IndexedFile
 		f.RetryCount = 0
-		err = rows.Scan(&f.Filepath, &f.Size, &f.SHA1)
+		err = rows.Scan(&f.Filepath, &f.Size, &f.CRC32)
 		if err != nil {
 			return nil, err
 		}
@@ -106,6 +121,10 @@ func (repo *SqliteRepo) ResetDownloadState() error {
 		return err
 	}
 	_, err = repo.db.Exec("UPDATE files SET taken = false")
+	if err != nil {
+		return err
+	}
+	_, err = repo.db.Exec("UPDATE empty_dirs SET done = false")
 	return err
 }
 

types.go

@@ -18,7 +18,7 @@ type UiFile struct {
 type IndexedFile struct {
 	Filepath   string `json:"path"`
 	Size       int64  `json:"size"`
-	SHA1       string `json:"sha1"`
+	CRC32      int    `json:"crc32"`
 	RetryCount int
 }