initial implementation rfc9111

Author: walterwanderley

README.md

@@ -19,11 +19,13 @@ sqlite3
 INSERT INTO temp.http_request VALUES('https://swapi.tech/api/films/1');
 
 # Fetch data from http_response table (created by the extension)
-SELECT * FROM http_response;
-
 SELECT JSON_EXTRACT(body, '$.result.properties.title') AS title,
   JSON_EXTRACT(body, '$.result.properties.release_date') AS release_date 
   FROM http_response;
+
+# Use cacheage, cachelifetime or cachexpired function to check cache validity based on RFC9111
+SELECT url, cacheage(header) AS age, cachelifetime(header, request_time, true) AS lifetime, cachexpired(header, request_time, true) AS expired 
+FROM http_response; 
 ```
 
 ## Configuring
@@ -34,7 +36,7 @@ You can configure the behaviour by passing parameters to a VIRTUAL TABLE.
 |-------|-------------|---------|
 | timeout | Timeout in milliseconds | 0 |
 | insecure | Insecure skip TLS validation | false |
-| ignore_status_error | Don't persist responses if status code != 2xx | false |
+| status_code | Comma-separated list of HTTP status code to persist. Use empty to persist all status | 200,301,404 |
 | response_table | Database table used to store response data | http_response |
 | oauth2_client_id | Oauth2 Client ID | |
 | oauth2_client_secret | Oauth2 Client Secret | |

cmd/sqlite-http-proxy/main.go

@@ -28,10 +28,11 @@ func main() {
 	allowHTTP2 := fs.BoolLong("h2", "Allow HTTP2")
 	ttl := fs.UintLong("ttl", 0, "Time to Live in seconds (0 is infinite time)")
 	responseTables := fs.StringListLong("response-table", "List of database tables used to store response data")
-	forceCreateTables := fs.BoolLong("force-create-tables", "Force create response tables if not exists")
 	caCert := fs.StringLong("ca-cert", "", "Path to CA Certificate file (required to HTTPS proxy)")
 	caCertKey := fs.StringLong("ca-cert-key", "", "Path to CA Certificate Key file (required to HTTPS proxy)")
 	readOnly := fs.BoolLong("ro", "Read Only mode. Do not store new HTTP responses")
+	rfc9111 := fs.BoolLong("rfc9111", "Use RFC9111 spec")
+	shared := fs.BoolLong("shared", "Enable shared cache mode")
 	_ = fs.String('c', "config", "", "config file (optional)")
 
 	if err := ff.Parse(fs, os.Args[1:],
@@ -49,8 +50,8 @@ func main() {
 	}
 
 	if *verbose {
-		fmt.Printf("Using options: port=%d db-params=%s, h2=%v, ttl=%d, response-tables=%v, force-create-tables=%v, ca-cert=%s, ca-cert-key=%s, read-only=%v\n",
-			*port, *dbParams, *allowHTTP2, *ttl, *responseTables, *forceCreateTables, *caCert, *caCertKey, *readOnly)
+		fmt.Printf("Using options: port=%d db-params=%s, h2=%v, ttl=%d, response-tables=%v, ca-cert=%s, ca-cert-key=%s, read-only=%v\n",
+			*port, *dbParams, *allowHTTP2, *ttl, *responseTables, *caCert, *caCertKey, *readOnly)
 	}
 
 	dbs := make([]*sql.DB, 0)
@@ -99,12 +100,11 @@ func main() {
 			}
 		} else {
 			tableList = *responseTables
-			if *forceCreateTables {
-				err := db.CreateResponseTables(sqlDB, tableList...)
-				if err != nil {
-					log.Fatalf("force create tables on DB %q: %v", dsn, err)
-				}
+			err := db.CreateResponseTables(sqlDB, tableList...)
+			if err != nil {
+				log.Fatalf("force create tables on DB %q: %v", dsn, err)
 			}
+
 		}
 	}
 	if len(dbs) == 1 {
@@ -140,18 +140,36 @@ func main() {
 		proxy.Logger.Printf("INFO: Starting HTTP Proxy...")
 	}
 
-	proxy.OnRequest().Do(&requestHandler{
-		querier:  repository,
-		verbose:  *verbose,
-		ttl:      *ttl,
-		readOnly: *readOnly,
-	})
-	if !*readOnly {
-		proxy.OnResponse().Do(&responseHandler{
-			writer:  repository,
-			verbose: *verbose,
+	if *rfc9111 {
+		proxy.OnRequest().Do(&requestRFC9111Handler{
+			shared:   *shared,
+			querier:  repository,
+			verbose:  *verbose,
+			readOnly: *readOnly,
+		})
+	} else {
+		proxy.OnRequest().Do(&requestHandler{
+			querier:  repository,
+			verbose:  *verbose,
+			ttl:      *ttl,
+			readOnly: *readOnly,
 		})
 	}
+	if !*readOnly {
+		if *rfc9111 {
+			proxy.OnResponse().Do(&responseRFC9111Handler{
+				shared:  *shared,
+				writer:  repository,
+				verbose: *verbose,
+			})
+		} else {
+			proxy.OnResponse().Do(&responseHandler{
+				writer:  repository,
+				verbose: *verbose,
+			})
+		}
+
+	}
 
 	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
 	if err != nil {

cmd/sqlite-http-proxy/request.go

@@ -12,6 +12,7 @@ import (
 	"github.com/elazarl/goproxy"
 
 	"github.com/walterwanderley/sqlite-http-cache/db"
+	cachehttp "github.com/walterwanderley/sqlite-http-cache/http"
 )
 
 type requestQuerier interface {
@@ -25,6 +26,12 @@ type requestHandler struct {
 	querier  requestQuerier
 }
 
+type userData struct {
+	requestTime time.Time
+	databaseID  int
+	tableName   string
+}
+
 func (h *requestHandler) Handle(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
 	if r.Method != http.MethodGet {
 		return r, nil
@@ -37,25 +44,33 @@ func (h *requestHandler) Handle(r *http.Request, ctx *goproxy.ProxyCtx) (*http.R
 			slog.Error("database query", "error", err.Error())
 		}
 		// tell the responseHandler to save the new response data
-		ctx.UserData = ":-1"
+		ctx.UserData = userData{
+			requestTime: time.Now(),
+			databaseID:  -1,
+		}
 		return r, nil
 	}
 
-	if !h.readOnly && h.ttl > 0 && uint(time.Since(resp.Timestamp).Seconds()) > h.ttl {
+	if !h.readOnly && h.ttl > 0 && uint(time.Since(resp.ResponseTime).Seconds()) > h.ttl {
 		// data is too old, tell the responseHandler to save the new data
-		ctx.UserData = fmt.Sprintf("%s:%d", resp.TableName, resp.DatabaseID)
+		ctx.UserData = userData{
+			requestTime: time.Now(),
+			databaseID:  resp.DatabaseID,
+			tableName:   resp.TableName,
+		}
 		return r, nil
 	}
 	if h.verbose {
-		slog.Info("serving from database", "url", url, "status", resp.Status, "timestamp", resp.Timestamp.Format(time.RFC3339))
+		slog.Info("serving from database", "url", url, "status", resp.Status, "request_time", resp.RequestTime.Format(time.RFC3339), "response_time", resp.ResponseTime.Format(time.RFC3339))
 	}
 
 	header := http.Header(resp.Header)
-	if date := header.Get("Date"); date != "" {
-		d, err := time.Parse(time.RFC1123, date)
-		if err == nil {
-			header.Set("Age", fmt.Sprint(int(time.Since(d).Seconds())))
-		}
+	if header.Get("Date") == "" {
+		header.Set("Date", time.Now().Format(time.RFC1123))
+	}
+	// TODO fix age calc based on RFC 9111
+	if age := cachehttp.Age(header); age != nil {
+		header.Set("Age", fmt.Sprint(*age))
 	}
 
 	return r, &http.Response{

cmd/sqlite-http-proxy/request_rfc9111.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"database/sql"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"time"
+
+	"github.com/elazarl/goproxy"
+
+	cachehttp "github.com/walterwanderley/sqlite-http-cache/http"
+)
+
+type requestRFC9111Handler struct {
+	shared   bool
+	verbose  bool
+	readOnly bool
+	querier  requestQuerier
+}
+
+func (h *requestRFC9111Handler) Handle(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
+	now := time.Now()
+	if r.Method != http.MethodGet {
+		return r, nil
+	}
+
+	cc := cachehttp.ParseCacheControl(r.Header, nil, h.shared)
+	if !cc.Cacheable() {
+		return r, nil
+	}
+	if h.shared && r.Header.Get("Authorization") != "" {
+		return r, nil
+	}
+
+	url := ctx.Req.URL.String()
+	resp, err := h.querier.FindByURL(r.Context(), url)
+	if err != nil {
+		if !errors.Is(err, sql.ErrNoRows) {
+			slog.Error("database query", "error", err.Error())
+		}
+
+		if !h.readOnly {
+			// tell the responseHandler to save the new response data
+			ctx.UserData = userData{
+				requestTime: now,
+				databaseID:  -1,
+			}
+		}
+		return r, nil
+	}
+
+	respCC := cachehttp.ParseCacheControl(http.Header(resp.Header), &now, h.shared)
+
+	if respCC.Expired() {
+		fmt.Println("AQUIIIII")
+		if !h.readOnly {
+			// data is too old, tell the responseHandler to save the new data
+			ctx.UserData = userData{
+				requestTime: now,
+				databaseID:  resp.DatabaseID,
+				tableName:   resp.TableName,
+			}
+		}
+		return r, nil
+	}
+	if h.verbose {
+		slog.Info("serving from database", "url", url, "status", resp.Status, "request_time", resp.RequestTime.Format(time.RFC3339), "response_time", resp.ResponseTime.Format(time.RFC3339))
+	}
+
+	header := http.Header(resp.Header)
+	if header.Get("Date") == "" {
+		header.Set("Date", time.Now().Format(time.RFC1123))
+	}
+	// TODO fix age calc based on RFC 9111
+	if age := cachehttp.Age(header); age != nil {
+		header.Set("Age", fmt.Sprint(*age))
+	}
+
+	return r, &http.Response{
+		StatusCode: resp.Status,
+		Body:       resp.Body,
+		Header:     header,
+	}
+}

cmd/sqlite-http-proxy/response.go

@@ -4,8 +4,7 @@ import (
 	"context"
 	"log/slog"
 	"net/http"
-	"strconv"
-	"strings"
+	"time"
 
 	"github.com/elazarl/goproxy"
 	"github.com/walterwanderley/sqlite-http-cache/db"
@@ -21,7 +20,8 @@ type responseHandler struct {
 }
 
 func (h *responseHandler) Handle(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
-	if ctx.UserData != nil {
+	ud, ok := ctx.UserData.(userData)
+	if ok {
 		if h.verbose {
 			slog.Info("recording response", "url", ctx.Req.URL.String(), "status", resp.StatusCode)
 		}
@@ -30,12 +30,10 @@ func (h *responseHandler) Handle(resp *http.Response, ctx *goproxy.ProxyCtx) *ht
 			slog.Error("adapter response body", "error", err)
 		} else {
 			go func() {
-				userData := ctx.UserData.(string)
-				tableName, databaseID, ok := strings.Cut(userData, ":")
-				if ok {
-					responseDB.DatabaseID, _ = strconv.Atoi(databaseID)
-				}
-				responseDB.TableName = tableName
+				responseDB.RequestTime = ud.requestTime
+				responseDB.ResponseTime = time.Now()
+				responseDB.DatabaseID = ud.databaseID
+				responseDB.TableName = ud.tableName
 				err := h.writer.Write(context.Background(), ctx.Req.URL.String(), responseDB)
 				if err != nil {
 					slog.Error("recording response", "error", err, "url", ctx.Req.URL.String(), "status", resp.StatusCode)

cmd/sqlite-http-proxy/response_rfc9111.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+	"net/http"
+	"time"
+
+	"github.com/elazarl/goproxy"
+	"github.com/walterwanderley/sqlite-http-cache/db"
+	cachehttp "github.com/walterwanderley/sqlite-http-cache/http"
+)
+
+type responseRFC9111Handler struct {
+	shared  bool
+	writer  responseWriter
+	verbose bool
+}
+
+func (h *responseRFC9111Handler) Handle(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
+	now := time.Now()
+	cc := cachehttp.ParseCacheControl(resp.Header, &now, h.shared)
+	if !cc.Cacheable() {
+		return resp
+	}
+
+	ud, ok := ctx.UserData.(userData)
+	if !ok {
+		return resp
+	}
+	responseDB, err := db.HttpToResponse(resp)
+	if err != nil {
+		slog.Error("adapter response body", "error", err)
+	} else {
+		if !cc.Expired() {
+			return resp
+		}
+		if h.verbose {
+			slog.Info("recording response", "url", ctx.Req.URL.String(), "status", resp.StatusCode)
+		}
+		go func() {
+			responseDB.RequestTime = ud.requestTime
+			responseDB.ResponseTime = time.Now()
+			responseDB.DatabaseID = ud.databaseID
+			responseDB.TableName = ud.tableName
+			err := h.writer.Write(context.Background(), ctx.Req.URL.String(), responseDB)
+			if err != nil {
+				slog.Error("recording response", "error", err, "url", ctx.Req.URL.String(), "status", resp.StatusCode)
+			}
+		}()
+	}
+
+	return resp
+}